From 197f3036d10e8746e98033c51fc3f5ecafea9b11 Mon Sep 17 00:00:00 2001 From: deajan Date: Thu, 12 Nov 2015 01:26:38 +0100 Subject: [PATCH] Initial v2.0 upload --- CHANGELOG.md | 33 +- CODING_STYLE.TXT | 147 ++ LICENCE.txt => LICENCE.TXT | 2 +- dev/debug_obackup.sh | 2236 ++++++++++++++++++++++++++ dev/merge.sh | 49 + dev/n_obackup.sh | 1352 ++++++++++++++++ dev/ofunctions.sh | 885 ++++++++++ exclude.list.example | 8 +- host_backup.conf | 131 +- install.sh | 78 +- obackup-batch.sh | 188 ++- obackup.sh | 3103 +++++++++++++++++++++--------------- ssh_filter.sh | 72 +- 13 files changed, 6749 insertions(+), 1535 deletions(-) create mode 100644 CODING_STYLE.TXT rename LICENCE.txt => LICENCE.TXT (95%) create mode 100755 dev/debug_obackup.sh create mode 100755 dev/merge.sh create mode 100755 dev/n_obackup.sh create mode 100644 dev/ofunctions.sh mode change 100644 => 120000 exclude.list.example diff --git a/CHANGELOG.md b/CHANGELOG.md index 13aa011..5d2be4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,3 @@ -SHORT FUTURE IMPROVEMENTS -------------------------- - -- Rewrite rsync exclude patterns using \"pattern\" instead of escaped chars -- Clean most of recursive task creation code - KNOWN ISSUES ------------ @@ -11,16 +5,27 @@ KNOWN ISSUES - Bandwidth parameter is ignored for SQL backups - Missing symlink support when run from MSYS environment -UNDER WORK ----------- - -- Commands like cp should have their stderr redirected to log file -- Mysqldump must be checked for not telling success if a table is damaged (also check for event table error) -- Mysqldump commands error msg must be logged - - CHANGELOG --------- + +README: FreeBSD execution needs mailer (not found), sudo missing, bash needed, sed missing (see if StripQuotes mandatory) + +! XX Dec 2015: obackup v2.0 released +- Added reverse backup, now backups can be local, pushed or pulled to or from a remote system +- Better fallback for SendAlert even if disk full +- Added an alert email sent on warnings while backup script is running +- Way better logging of errors in _GetDirectoriesSizeX, _BackupDatabaseX, _CreateStorageDirectoriesX +- Added bogus config file checks & environment checks +- Full code refactoring to use local and remote code once +- Fully merged codebase with osync + - Added (much) more verbose debugging (and possibility to remove debug code to gain speed) + - Replace child_pid by $? directly, add a better sub process killer in TrapQuit + - Added some automatic checks in code, for _DEBUG mode (and _PARANOIA_DEBUG now) + - Improved Logging + - Updated obackup to be fully compliant with coding style +- A long list of minor improvements + +v0-1.x - Jan 2013 - Oct 2015 - New function to kill child processes - Fixed no_maxtime not honored - Improved some logging, also added highlighting to stdout errors diff --git a/CODING_STYLE.TXT b/CODING_STYLE.TXT new file mode 100644 index 0000000..c0e89f2 --- /dev/null +++ b/CODING_STYLE.TXT @@ -0,0 +1,147 @@ +Coding style used for my bash projects (v2.1 Oct 2015) + +++++++ Header + +Always use the following header + +----BEGIN HEADER +#!/usr/bin/env bash + +PROGRAM="program-name" # Long description +AUTHOR="(L) 20XX-20YY by Orsiris \"Ozy\" de Jong" +CONTACT="http://www.example.com me@example.com" +PROGRAM_BUILD=YYYYMMDDVV + +## Optional instructions +----END HEADER + +Using bind style versionning: +YYYYMMDDVV (Year, Month, Day, Revision): Example: 2015012402 = 2nd revision of 24 Jan 2015 + +#!/usr/bin/env bash instead of #!/bin/bash + +Change old scripts with +for i in $(grep -r '#!/bin/bash' * |cut -f1 -d':'); do sed -i 's&#!/bin/bash&#!/usr/bin/env bash&g' $i; done + + +type instead of type -p for bash test (other shells don't know -p) +++++++ Indentation + +Using tabs +Transform old shell scripts using unexpand command + +++++++ Comments + +Some command # comment +## Some comment on a new line +################################################# Some separation + +++++++ Work comments + +Whenever there is some idea to postpone, use #TODO[-version]:[dev-name:] some remark +A marker must be left where on the line a dev is working (when the work isn't finished). Marker is #WIP:dev-name: some remark +dev-name is mandatory if more than one person is coding +Example: #TODO-v2.1:deajan: need to do something + +++++++ Variables + +All local variables are lowercase, separated by _ (ex: low_wait) +All global variables full upercase, separated by _ (ex: EXEC_TIME) +All environment variables (verbose, silent, debug, etc) have prefix _ and are full upercase, separated by _ (ex: _PARANOIA_DEBUG) + +++++++ Functions + +Every word in a function begins with an uppercase (ex: SomeFunctionDoesThings) + +Define functions this way. Use sed ':a;N;$!ba;s/\n{\n/ {\n/g' to adapt when opening bracket is on a new line. + +function something { + +} + +If function has some arguments, use local variable names that are more readable than $1...$n. Explain via comments what those variables contain if needed. + +function anotherthing { + local var_name="${1}" + local other_var_name="${2}" # This variable contains stuff +} + +Functions should always have return status +function thirdthing { + some_command + return $? +} + +++++++ Sub functions + +When a function is a subroutine of another function, it is called _SomethingAsSubFunction + +++++++ Function argument check + +Bash does not provide any checks against missing function arguments. Also, missing quotes can lead to an inconsistent number of arguments. +Every function call will be checked by __CheckArguments which takes the number of arguments, $# (the real number of args given), the parent function name and the parent function's arguments. +__CheckArguments will trigger a critical error if number of arguments if incorrect. This will also prevent silent typo errors. +Ex: + +function Something { + local some="${1}" + local other="${2}" + local args="${3}" + __CheckArguments 3 $# $FUNCNAME "$*" + +__CheckArguments will only trigger if script is called with DEBUG=yes +Also, with PARANOIA_DEBUG=yes, __CheckArguments will recount all arguments given by "$*" and compare. This can mislead if arguments contain spaces. + +++++++ If statements + +If statements will be fully written (word "if" must be used). then is written on the same line. +(Use sed ':a;N;$!ba;s/]\n\t*then/]; then/g' to convert files to this format... Replace "],new line, zero or more tabs, then" by "; then") +if [ something ]; then + stuff +else + other stuff +fi + +++++++ Logging + +A logging function is available with the following levels of logging: + +- DEBUG: Only log this when DEBUG flas is set in program. Any command forged for eval should be logged by this. +- NOTICE: Standard messages +- WARN: Requires attention +- ERROR: Program produced an error but continues execution +- CRITICAL: Program execution is halted + +++++++ Eval + +Most commands should be logged to a tmp file. +The basic way of doing is: + +cmd='"something '$somevar'" > some_file 2>&1' +eval $cmd & +WaitForTaskCompletion $! 0 0 $FUNCNAME + +Remote commands should exist as: + +cmd=$SSH_CMD' "some; commands \"'$VARIABLE'\" some; other; commands" > some_file 2>&1' + +++++++ File variables + +All eval cmd should exit their content to a file called "$RUNDIR/osync.$FUNCNAME.$SCRIPT_PID" +Dots are used instead of '_' so variables can be separated with a forbidden char in variables, so they get detected. + +++++++ Finding code errors + +Use shellcheck.net now and then (ignore SC2086 in our case) + +Use a low tech approach to find uneven number of quotes per line + +tr -cd "'\n" < my_bash_file.sh | awk 'length%2==1 {print NR, $0}' +tr -cd "\"\n" < my_bash_file.sh | awk 'length%2==1 {print NR, $0}' + +++++++ ofunctions + +As obackup and osync share alot of common functions, ofunctions.sh will host all shared code. +Dev programs n_osync.sh and n_obackup.sh will source ofunctions.sh +Release programs will still include ofunctions.sh in order to enhance ease of use. + diff --git a/LICENCE.txt b/LICENCE.TXT similarity index 95% rename from LICENCE.txt rename to LICENCE.TXT index a631274..0359985 100644 --- a/LICENCE.txt +++ b/LICENCE.TXT @@ -1,4 +1,4 @@ -Copyright (c) 2013, Orsiris "Ozy" de Jong. ozy@netpower.fr +Copyright (c) 2013-2015, Orsiris "Ozy" de Jong. ozy@netpower.fr All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/dev/debug_obackup.sh b/dev/debug_obackup.sh new file mode 100755 index 0000000..4c1dc58 --- /dev/null +++ b/dev/debug_obackup.sh @@ -0,0 +1,2236 @@ +#!/usr/bin/env bash + +###### Remote push/pull (or local) backup script for files & databases +###### (L) 2013-2015 by Orsiris "Ozy" de Jong (www.netpower.fr) +PROGRAM="obackup" +AUTHOR="(L) 2013-2015 by Orsiris de Jong" +CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" +PROGRAM_VERSION=2.0-pre +PROGRAM_BUILD=2015111107 +IS_STABLE=no + +FUNC_BUILD=2015111102 +## BEGIN Generic functions for osync & obackup written in 2013-2015 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr + +## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode +if ! type "$BASH" > /dev/null; then + echo "Please run this script only with bash shell. Tested on bash >= 3.2" + exit 127 +fi + +# Environment variables +_DRYRUN=0 +_SILENT=0 + +# Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags +ERROR_ALERT=0 +WARN_ALERT=0 + +## allow function call checks #__WITH_PARANOIA_DEBUG +if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG + _DEBUG=yes #__WITH_PARANOIA_DEBUG +fi #__WITH_PARANOIA_DEBUG + +## allow debugging from command line with _DEBUG=yes +if [ ! "$_DEBUG" == "yes" ]; then + _DEBUG=no + SLEEP_TIME=.1 + _VERBOSE=0 +else + SLEEP_TIME=1 + trap 'TrapError ${LINENO} $?' ERR + _VERBOSE=1 +fi + +SCRIPT_PID=$$ + +LOCAL_USER=$(whoami) +LOCAL_HOST=$(hostname) + +## Default log file until config file is loaded +if [ -w /var/log ]; then + LOG_FILE="/var/log/$PROGRAM.log" +else + LOG_FILE="./$PROGRAM.log" +fi + +## Default directory where to store temporary run files +if [ -w /tmp ]; then + RUN_DIR=/tmp +elif [ -w /var/tmp ]; then + RUN_DIR=/var/tmp +else + RUN_DIR=. +fi + +## Log a state message every $KEEP_LOGGING seconds. Should not be equal to soft or hard execution time so your log will not be unnecessary big. +KEEP_LOGGING=1801 + +## Correct output of sort command (language agnostic sorting) +export LC_ALL=C + +# Standard alert mail body +MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." + +# Default alert attachment filename +ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log" + +# Set error exit code if a piped command fails + set -o pipefail + set -o errtrace + + +function Dummy { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + sleep .1 +} + +function _Logger { + local svalue="${1}" # What to log to screen + local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value + echo -e "$lvalue" >> "$LOG_FILE" + + if [ $_SILENT -eq 0 ]; then + echo -e "$svalue" + fi +} + +function Logger { + local value="${1}" # Sentence to log (in double quotes) + local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL + + # Special case in daemon mode we should timestamp instead of counting seconds + if [ "$sync_on_changes" == "1" ]; then + prefix="$(date) - " + else + prefix="TIME: $SECONDS - " + fi + # + + if [ "$level" == "CRITICAL" ]; then + _Logger "$prefix\e[41m$value\e[0m" "$prefix$value" + ERROR_ALERT=1 + return + elif [ "$level" == "ERROR" ]; then + _Logger "$prefix\e[91m$value\e[0m" "$prefix$value" + ERROR_ALERT=1 + return + elif [ "$level" == "WARN" ]; then + _Logger "$prefix\e[93m$value\e[0m" "$prefix$value" + WARN_ALERT=1 + return + elif [ "$level" == "NOTICE" ]; then + _Logger "$prefix$value" + return + elif [ "$level" == "DEBUG" ]; then + if [ "$_DEBUG" == "yes" ]; then + _Logger "$prefix$value" + return + fi + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG + _Logger "$prefix$value" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG + else + _Logger "\e[41mLogger function called without proper loglevel.\e[0m" + _Logger "$prefix$value" + fi +} + +# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X +function KillChilds { + local pid="${1}" + local self="${2:-false}" + + if children="$(pgrep -P "$pid")"; then + for child in $children; do + KillChilds "$child" true + done + fi + + # Try to kill nicely, if not, wait 30 seconds to let Trap actions happen before killing + if [ "$self" == true ]; then + kill -s SIGTERM "$pid" || (sleep 30 && kill -9 "$pid" &) + fi +} + +function TrapError { + local job="$0" + local line="$1" + local code="${2:-1}" + if [ $_SILENT -eq 0 ]; then + echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}" + fi +} + +function Spinner { + if [ $_SILENT -eq 1 ]; then + return 0 + fi + + case $toggle + in + 1) + echo -n " \ " + echo -ne "\r" + toggle="2" + ;; + + 2) + echo -n " | " + echo -ne "\r" + toggle="3" + ;; + + 3) + echo -n " / " + echo -ne "\r" + toggle="4" + ;; + + *) + echo -n " - " + echo -ne "\r" + toggle="1" + ;; + esac +} + +function SedStripQuotes { + echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") +} + +function StripSingleQuotes { + local string="${1}" + string="${string/#\'/}" # Remove singlequote if it begins string + string="${string/%\'/}" # Remove singlequote if it ends string + echo "$string" +} + +function StripDoubleQuotes { + local string="${1}" + string="${string/#\"/}" + string="${string/%\"/}" + echo "$string" +} + +function StripQuotes { + local string="${1}" + echo "$(StripSingleQuotes $(StripDoubleQuotes $string))" +} + +function EscapeSpaces { + local string="${1}" # String on which spaces will be escaped + echo "${string// /\ }" +} + +function IsNumeric { + eval "local value=\"${1}\"" # Needed so variable variables can be processed + + local re="^-?[0-9]+([.][0-9]+)?$" + if [[ $value =~ $re ]]; then + echo 1 + else + echo 0 + fi +} + +function CleanUp { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$_DEBUG" != "yes" ]; then + rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID" + fi +} + +function SendAlert { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$_DEBUG" == "yes" ]; then + Logger "Debug mode, no warning email will be sent." "NOTICE" + return 0 + fi + + # + if [ "$_QUICK_SYNC" == "2" ]; then + Logger "Current task is a quicksync task. Will not send any alert." "NOTICE" + return 0 + fi + # + + eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE" + MAIL_ALERT_MSG="$MAIL_ALERT_MSG"$'\n\n'$(tail -n 25 "$LOG_FILE") + if [ $ERROR_ALERT -eq 1 ]; then + subject="Error alert for $INSTANCE_ID" + elif [ $WARN_ALERT -eq 1 ]; then + subject="Warning alert for $INSTANCE_ID" + else + subject="Alert for $INSTANCE_ID" + fi + + # Need better fallback if mail sending does not succeed + if type mutt > /dev/null 2>&1 ; then + echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS -a "$ALERT_LOG_FILE" + if [ $? != 0 ]; then + Logger "WARNING: Cannot send alert email via $(type -p mutt) !!!" "WARN" + else + Logger "Sent alert mail using mutt." "NOTICE" + return 0 + fi + fi + + if type mail > /dev/null 2>&1 ; then + echo "$MAIL_ALERT_MSG" | $(type -p mail) -a "$ALERT_LOG_FILE" -s "$subject" $DESTINATION_MAILS + if [ $? != 0 ]; then + Logger "WARNING: Cannot send alert email via $(type -p mail) with attachments !!!" "WARN" + echo "$MAIL_ALERT_MSG" | $(type -p mail) -s "$subject" $DESTINATION_MAILS + if [ $? != 0 ]; then + Logger "WARNING: Cannot send alert email via $(type -p mail) without attachments !!!" "WARN" + else + Logger "Sent alert mail using mail command without attachment." "NOTICE" + return 0 + fi + else + Logger "Sent alert mail using mail command." "NOTICE" + return 0 + fi + fi + + if type sendmail > /dev/null 2>&1 ; then + echo -e "$subject\r\n$MAIL_ALERT_MSG" | $(type -p sendmail) $DESTINATION_MAILS + if [ $? != 0 ]; then + Logger "WARNING: Cannot send alert email via $(type -p sendmail) !!!" "WARN" + else + Logger "Sent alert mail using sendmail command without attachment." "NOTICE" + return 0 + fi + fi + + if type sendemail > /dev/null 2>&1 ; then + if [ "$SMTP_USER" != "" ] && [ "$SMTP_PASSWORD" != "" ]; then + SMTP_OPTIONS="-xu $SMTP_USER -xp $SMTP_PASSWORD" + else + SMTP_OPTIONS="" + fi + $(type -p sendemail) -f $SENDER_MAIL -t $DESTINATION_MAILS -u "$subject" -m "$MAIL_ALERT_MSG" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1 + if [ $? != 0 ]; then + Logger "WARNING: Cannot send alert email via $(type -p sendemail) !!!" "WARN" + else + Logger "Sent alert mail using sendemail command without attachment." "NOTICE" + return 0 + fi + fi + + # If function has not returned 0 yet, assume it's critical that no alert can be sent + Logger "/!\ CRITICAL: Cannot send alert" "ERROR" # Is not marked critical because execution must continue + + # Delete tmp log file + if [ -f "$ALERT_LOG_FILE" ]; then + rm "$ALERT_LOG_FILE" + fi +} + +function LoadConfigFile { + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + local config_file="${1}" + + if [ ! -f "$config_file" ]; then + Logger "Cannot load configuration file [$config_file]. Cannot start." "CRITICAL" + exit 1 + elif [[ "$1" != *".conf" ]]; then + Logger "Wrong configuration file supplied [$config_file]. Cannot start." "CRITICAL" + exit 1 + else + grep '^[^ ]*=[^;&]*' "$config_file" > "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" # WITHOUT COMMENTS + # Shellcheck source=./sync.conf + source "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" + fi +} + +function GetLocalOS { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + local local_os_var=$(uname -spio 2>&1) + if [ $? != 0 ]; then + local local_os_var=$(uname -v 2>&1) + if [ $? != 0 ]; then + local local_os_var=($uname) + fi + fi + + case $local_os_var in + *"Linux"*) + LOCAL_OS="Linux" + ;; + *"BSD"*) + LOCAL_OS="BSD" + ;; + *"MINGW32"*) + LOCAL_OS="msys" + ;; + *"Darwin"*) + LOCAL_OS="MacOSX" + ;; + *) + Logger "Running on >> $local_os_var << not supported. Please report to the author." "ERROR" + exit 1 + ;; + esac + Logger "Local OS: [$local_os_var]." "DEBUG" +} + +function GetRemoteOS { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$REMOTE_OPERATION" == "yes" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + local cmd=$SSH_CMD' "uname -spio" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 120 240 $FUNCNAME"-1" + retval=$? + if [ $retval != 0 ]; then + local cmd=$SSH_CMD' "uname -v" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 120 240 $FUNCNAME"-2" + retval=$? + if [ $retval != 0 ]; then + local cmd=$SSH_CMD' "uname" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 120 240 $FUNCNAME"-3" + retval=$? + if [ $retval != 0 ]; then + Logger "Cannot Get remote OS type." "ERROR" + fi + fi + fi + + local remote_os_var=$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID) + + case $remote_os_var in + *"Linux"*) + REMOTE_OS="Linux" + ;; + *"BSD"*) + REMOTE_OS="BSD" + ;; + *"MINGW32"*) + REMOTE_OS="msys" + ;; + *"Darwin"*) + REMOTE_OS="MacOSX" + ;; + *"ssh"*|*"SSH"*) + Logger "Cannot connect to remote system." "CRITICAL" + exit 1 + ;; + *) + Logger "Running on remote OS failed. Please report to the author if the OS is not supported." "CRITICAL" + Logger "Remote OS said:\n$remote_os_var" "CRITICAL" + exit 1 + esac + + Logger "Remote OS: [$remote_os_var]." "DEBUG" + fi +} + +function WaitForTaskCompletion { + local pid="${1}" # pid to wait for + local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. + local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. + local caller_name="${4}" # Who called this function + Logger "$FUNCNAME called by [$caller_name]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + __CheckArguments 4 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once + local log_ttime=0 # local time instance for comparaison + + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function + + while eval "$PROCESS_TEST_CMD" > /dev/null + do + Spinner + exec_time=$(($SECONDS - $seconds_begin)) + if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then + if [ $log_ttime -ne $exec_time ]; then + log_ttime=$exec_time + Logger "Current task still running." "NOTICE" + fi + fi + if [ $exec_time -gt $soft_max_time ]; then + if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then + Logger "Max soft execution time exceeded for task." "WARN" + soft_alert=1 + SendAlert + + fi + if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then + Logger "Max hard execution time exceeded for task. Stopping task execution." "ERROR" + kill -s SIGTERM $pid + if [ $? == 0 ]; then + Logger "Task stopped succesfully" "NOTICE" + else + Logger "Sending SIGTERM to proces failed. Trying the hard way." "ERROR" + sleep 5 && kill -9 $pid + if [ $? != 0 ]; then + Logger "Could not stop task." "ERROR" + fi + fi + return 1 + fi + fi + sleep $SLEEP_TIME + done + wait $pid + local retval=$? + Logger "$FUNCNAME ended for [$caller_name] with status $retval." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + return $retval +} + +function WaitForCompletion { + local pid="${1}" # pid to wait for + local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. + local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. + local caller_name="${4}" # Who called this function + Logger "$FUNCNAME called by [$caller_name]" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + __CheckArguments 4 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once + local log_ttime=0 # local time instance for comparaison + + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function + + while eval "$PROCESS_TEST_CMD" > /dev/null + do + Spinner + if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ]; then + if [ $log_time -ne $SECONDS ]; then + log_time=$SECONDS + Logger "Current task still running." "NOTICE" + fi + fi + if [ $SECONDS -gt $soft_max_time ]; then + if [ $soft_alert -eq 0 ] && [ $soft_max_time != 0 ]; then + Logger "Max soft execution time exceeded for script." "WARN" + soft_alert=1 + SendAlert + fi + if [ $SECONDS -gt $hard_max_time ] && [ $hard_max_time != 0 ]; then + Logger "Max hard execution time exceeded for script. Stopping current task execution." "ERROR" + kill -s SIGTERM $pid + if [ $? == 0 ]; then + Logger "Task stopped succesfully" "NOTICE" + else + Logger "Sending SIGTERM to proces failed. Trying the hard way." "ERROR" + kill -9 $pid + if [ $? != 0 ]; then + Logger "Could not stop task." "ERROR" + fi + fi + return 1 + fi + fi + sleep $SLEEP_TIME + done + wait $pid + retval=$? + Logger "$FUNCNAME ended for [$caller_name] with status $retval." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + return $retval +} + +function RunLocalCommand { + local command="${1}" # Command to run + local hard_max_time="${2}" # Max time to wait for command to compleet + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ $_DRYRUN -ne 0 ]; then + Logger "Dryrun: Local command [$command] not run." "NOTICE" + return 1 + fi + Logger "Running command [$command] on local host." "NOTICE" + eval "$command" > "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" 2>&1 & + WaitForTaskCompletion $! 0 $hard_max_time $FUNCNAME + retval=$? + if [ $retval -eq 0 ]; then + Logger "Command succeded." "NOTICE" + else + Logger "Command failed." "ERROR" + fi + + if [ $_VERBOSE -eq 1 ] || [ $retval -ne 0 ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE" + fi + + if [ "$STOP_ON_CMD_ERROR" == "yes" ] && [ $retval -ne 0 ]; then + Logger "Stopping on command execution error." "CRITICAL" + exit 1 + fi +} + +## Runs remote command $1 and waits for completition in $2 seconds +function RunRemoteCommand { + local command="${1}" # Command to run + local hard_max_time="${2}" # Max time to wait for command to compleet + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + if [ $_DRYRUN -ne 0 ]; then + Logger "Dryrun: Local command [$command] not run." "NOTICE" + return 1 + fi + Logger "Running command [$command] on remote host." "NOTICE" + cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 0 $hard_max_time $FUNCNAME + retval=$? + if [ $retval -eq 0 ]; then + Logger "Command succeded." "NOTICE" + else + Logger "Command failed." "ERROR" + fi + + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ] && ([ $_VERBOSE -eq 1 ] || [ $retval -ne 0 ]) + then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE" + fi + + if [ "$STOP_ON_CMD_ERROR" == "yes" ] && [ $retval -ne 0 ]; then + Logger "Stopping on command execution error." "CRITICAL" + exit 1 + fi +} + +function RunBeforeHook { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then + RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE + fi + + if [ "$REMOTE_RUN_BEFORE_CMD" != "" ]; then + RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE + fi +} + +function RunAfterHook { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$LOCAL_RUN_AFTER_CMD" != "" ]; then + RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER + fi + + if [ "$REMOTE_RUN_AFTER_CMD" != "" ]; then + RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER + fi +} + +function CheckConnectivityRemoteHost { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug + + if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then + eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & + WaitForTaskCompletion $! 180 180 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot ping $REMOTE_HOST" "CRITICAL" + return 1 + fi + fi + fi +} + +function CheckConnectivity3rdPartyHosts { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug + + if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then + remote_3rd_party_success=0 + OLD_IFS=$IFS + IFS=$' \t\n' + for i in $REMOTE_3RD_PARTY_HOSTS + do + eval "$PING_CMD $i > /dev/null 2>&1" & + WaitForTaskCompletion $! 360 360 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot ping 3rd party host $i" "WARN" + else + remote_3rd_party_success=1 + fi + done + IFS=$OLD_IFS + if [ $remote_3rd_party_success -ne 1 ]; then + Logger "No remote 3rd party host responded to ping. No internet ?" "CRITICAL" + return 1 + fi + fi + fi +} + +#__BEGIN_WITH_PARANOIA_DEBUG +function __CheckArguments { + # Checks the number of arguments of a function and raises an error if some are missing + + if [ "$_DEBUG" == "yes" ]; then + local number_of_arguments="${1}" # Number of arguments the tested function should have + local number_of_given_arguments="${2}" # Number of arguments that have been passed + local function_name="${3}" # Function name that called __CheckArguments + + if [ "$_PARANOIA_DEBUG" == "yes" ]; then + Logger "Entering function [$function_name]." "DEBUG" + fi + + # All arguments of the function to check are passed as array in ${4} (the function call waits for $@) + # If any of the arguments contains spaces, bash things there are two aguments + # In order to avoid this, we need to iterate over ${4} and count + + local iterate=4 + local fetch_arguments=1 + local arg_list="" + while [ $fetch_arguments -eq 1 ]; do + cmd='argument=${'$iterate'}' + eval $cmd + if [ "$argument" = "" ]; then + fetch_arguments=0 + else + arg_list="$arg_list [Argument $(($iterate-3)): $argument]" + iterate=$(($iterate+1)) + fi + done + local counted_arguments=$((iterate-4)) + + if [ $counted_arguments -ne $number_of_arguments ]; then + Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $counted_arguments, see log file." "ERROR" + Logger "Arguments passed: $arg_list" "ERROR" + fi + fi +} + + +function old__CheckArguments { + # Checks the number of arguments and raises an error if some are missing + if [ "$_DEBUG" == "yes" ]; then + + local number_of_arguments="${1}" # Number of arguments a function should have + local number_of_given_arguments="${2}" # Number of arguments that have been passed + local function_name="${3}" # Function name that called __CheckArguments + local arguments="${4}" # All other arguments + + if [ "$_PARANOIA_DEBUG" == "yes" ]; then + Logger "Entering function [$function_name]." "DEBUG" + + # Paranoia check... Can help finding empty arguments. __CheckArguments should be grepped out in production builds. + local count=-3 # Number of arguments minus the function calls for __CheckArguments + for i in $@; do + count=$((count + 1)) + done + if [ $count -ne $1 ]; then + Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $count, see log file." "WARN" + echo "Argument list (including checks): $*" >> "$LOG_FILE" + fi + fi + + if [ $number_of_arguments -ne $number_of_given_arguments ]; then + Logger "Inconsistnent number of arguments in $function_name. Should have $number_of_arguments arguments, has $number_of_given_arguments arguments, see log file." "CRITICAL" + # Cannot user Logger here because $@ is a list of arguments + echo "Argumnt list: $4" >> "$LOG_FILE" + fi + + fi +} +#__END_WITH_PARANOIA_DEBUG + +function PreInit { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + ## SSH compression + if [ "$SSH_COMPRESSION" != "no" ]; then + SSH_COMP=-C + else + SSH_COMP= + fi + + ## Support for older config files without RSYNC_EXECUTABLE option + if [ "$RSYNC_EXECUTABLE" == "" ]; then + RSYNC_EXECUTABLE=rsync + fi + + ## Sudo execution option + if [ "$SUDO_EXEC" == "yes" ]; then + if [ "$RSYNC_REMOTE_PATH" != "" ]; then + RSYNC_PATH="sudo $RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" + else + RSYNC_PATH="sudo $RSYNC_EXECUTABLE" + fi + COMMAND_SUDO="sudo" + else + if [ "$RSYNC_REMOTE_PATH" != "" ]; then + RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" + else + RSYNC_PATH="$RSYNC_EXECUTABLE" + fi + COMMAND_SUDO="" + fi + + ## Set rsync default arguments + RSYNC_ARGS="-rlptgoD" + + if [ "$PRESERVE_ACL" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -A" + fi + if [ "$PRESERVE_XATTR" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -X" + fi + if [ "$RSYNC_COMPRESS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -z" + fi + if [ "$COPY_SYMLINKS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -L" + fi + if [ "$KEEP_DIRLINKS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -K" + fi + if [ "$PRESERVE_HARDLINKS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -H" + fi + if [ "$CHECKSUM" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --checksum" + fi + if [ $_DRYRUN -eq 1 ]; then + RSYNC_ARGS=$RSYNC_ARGS" -n" + DRY_WARNING="/!\ DRY RUN" + fi + if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" + fi + + ## Set compression executable and extension + COMPRESSION_LEVEL=3 + if type xz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| xz -$COMPRESSION_LEVEL" + COMPRESSION_EXTENSION=.xz + elif type lzma > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| lzma -$COMPRESSION_LEVEL" + COMPRESSION_EXTENSION=.lzma + elif type pigz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| pigz -$COMPRESSION_LEVEL" + COMPRESSION_EXTENSION=.gz + COMPRESSION_OPTIONS=--rsyncable + elif type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -$COMPRESSION_LEVEL" + COMPRESSION_EXTENSION=.gz + COMPRESSION_OPTIONS=--rsyncable + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" +} + +function PostInit { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + # Define remote commands + SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" + SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" + RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -p $REMOTE_PORT" +} + +function InitLocalOSSettings { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + ## If running under Msys, some commands do not run the same way + ## Using mingw version of find instead of windows one + ## Getting running processes is quite different + ## Ping command is not the same + if [ "$LOCAL_OS" == "msys" ]; then + FIND_CMD=$(dirname $BASH)/find + #TODO: The following command needs to be checked on msys. Does the $1 variable substitution work ? + # PROCESS_TEST_CMD assumes there is a variable $pid + PROCESS_TEST_CMD='ps -a | awk "{\$1=\$1}\$1" | awk "{print \$1}" | grep $pid' + PING_CMD="ping -n 2" + else + FIND_CMD=find + # PROCESS_TEST_CMD assumes there is a variable $pid + PROCESS_TEST_CMD='ps -p$pid' + PING_CMD="ping -c 2 -i .2" + fi + + ## Stat command has different syntax on Linux and FreeBSD/MacOSX + if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then + STAT_CMD="stat -f \"%Sm\"" + else + STAT_CMD="stat --format %y" + fi +} + +function InitRemoteOSSettings { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -E" + fi + + if [ "$REMOTE_OS" == "msys" ]; then + REMOTE_FIND_CMD=$(dirname $BASH)/find + else + REMOTE_FIND_CMD=find + fi +} + +## END Generic functions + +## Working directory for partial downloads +PARTIAL_DIR=".obackup_workdir_partial" + +# List of runtime created global variables +# $SQL_DISK_SPACE, disk space available on target for sql backups +# $FILE_DISK_SPACE, disk space available on target for file backups +# $SQL_BACKUP_TASKS, list of all databases to backup, space separated +# $SQL_EXCLUDED_TASKS, list of all database to exclude from backup, space separated +# $FILE_BACKUP_TASKS list of directories to backup, found in config file +# $FILE_RECURSIVE_BACKUP_TASKS, list of directories to backup, computed from config file recursive list +# $FILE_RECURSIVE_EXCLUDED_TASKS, list of all directories excluded from recursive list +# $FILE_SIZE_LIST, list of all directories to include in GetDirectoriesSize + +CAN_BACKUP_SQL=1 +CAN_BACKUP_FILES=1 + +function TrapStop { + Logger " /!\ WARNING: Manual exit of backup script. Backups may be in inconsistent state." "WARN" + exit 1 +} + +function TrapQuit { + if [ $ERROR_ALERT -ne 0 ]; then + SendAlert + CleanUp + Logger "Backup script finished with errors." "ERROR" + elif [ $WARN_ALERT -ne 0 ]; then + SendAlert + CleanUp + Logger "Backup script finished with warnings." "WARN" + else + CleanUp + Logger "Backup script finshed." "NOTICE" + fi + + KillChilds $$ > /dev/null 2>&1 +} + +function CheckEnvironment { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$REMOTE_OPERATION" == "yes" ]; then + if ! type ssh > /dev/null 2>&1 ; then + Logger "ssh not present. Cannot start backup." "CRITICAL" + exit 1 + fi + + if [ "$SQL_BACKUP" != "no" ]; then + if ! type mysqldump > /dev/null 2>&1 ; then + Logger "mysqldump not present. Cannot backup SQL." "CRITICAL" + CAN_BACKUP_SQL=0 + fi + if ! type mysql > /dev/null 2>&1 ; then + Logger "mysql not present. Cannot backup SQL." "CRITICAL" + CAN_BACKUP_SQL=0 + fi + fi + fi + + if [ "$FILE_BACKUP" != "no" ]; then + if [ "$ENCRYPTION" == "yes" ]; then + if ! type duplicity > /dev/null 2>&1 ; then + Logger "duplicity not present. Cannot backup encrypted files." "CRITICAL" + CAN_BACKUP_FILES=0 + fi + else + if ! type rsync > /dev/null 2>&1 ; then + Logger "rsync not present. Cannot backup files." "CRITICAL" + CAN_BACKUP_FILES=0 + fi + fi + fi +} + +function CheckCurrentConfig { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$INSTANCE_ID" == "" ]; then + Logger "No INSTANCE_ID defined in config file." "CRITICAL" + exit 1 + fi + + # Check all variables that should contain "yes" or "no" + declare -a yes_no_vars=(SQL_BACKUP FILE_BACKUP ENCRYPTION CREATE_DIRS KEEP_ABSOLUTE_PATHS GET_BACKUP_SIZE SUDO_EXEC SSH_COMPRESSION REMOTE_HOST_PING DATABASES_ALL PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS RSYNC_COMPRESS PARTIAL DELETE_VANISHED_FILES DELTA_COPIES ROTATE_SQL_BACKUPS ROTATE_FILE_BACKUPS STOP_ON_CMD_ERROR) + for i in ${yes_no_vars[@]}; do + test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file.\" \"CRITICAL\"; exit 1; fi" + eval "$test" + done + + if [ "$BACKUP_TYPE" != "local" ] && [ "$BACKUP_TYPE" != "pull" ] && [ "$BACKUP_TYPE" != "push" ]; then + Logger "Bogus BACKUP_TYPE value in config file." "CRITICAL" + exit 1 + fi + + # Check all variables that should contain a numerical value >= 0 + declare -a num_vars=(BACKUP_SIZE_MINIMUM BANDWIDTH SQL_WARN_MIN_SPACE FILE_WARN_MIN_SPACE SOFT_MAX_EXEC_TIME_DB_TASK HARD_MAX_EXEC_TIME_DB_TASK COMPRESSION_LEVEL SOFT_MAX_EXEC_TIME_FILE_TASK HARD_MAX_EXEC_TIME_FILE_TASK SOFT_MAX_EXEC_TIME_TOTAL HARD_MAX_EXEC_TIME_TOTAL ROTATE_COPIES MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER) + for i in ${num_vars[@]}; do + test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file.\" \"CRITICAL\"; exit 1; fi" + eval "$test" + done + + #TODO-v2.1: Add runtime variable tests (RSYNC_ARGS etc) +} + +function _ListDatabasesLocal { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + sql_cmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + Logger "cmd: $sql_cmd" "DEBUG" + eval "$sql_cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + if [ $? -eq 0 ]; then + Logger "Listing databases succeeded." "NOTICE" + else + Logger "Listing databases failed." "ERROR" + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + return 1 + fi + +} + +function _ListDatabasesRemote { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + sql_cmd="$SSH_CMD \"mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > \"$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID\" 2>&1" + Logger "cmd: $sql_cmd" "DEBUG" + eval "$sql_cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + if [ $? -eq 0 ]; then + Logger "Listing databases succeeded." "NOTICE" + else + Logger "Listing databases failed." "ERROR" + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + return 1 + fi +} + +function ListDatabases { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + local output_file # Return of subfunction + + if [ $CAN_BACKUP_SQL -ne 1 ]; then + Logger "Cannot list databases." "ERROR" + return 1 + fi + + Logger "Listing databases." "NOTICE" + + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]; then + _ListDatabasesLocal + if [ $? != 0 ]; then + output_file="" + else + output_file="$RUN_DIR/$PROGRAM._ListDatabasesLocal.$SCRIPT_PID" + fi + elif [ "$BACKUP_TYPE" == "pull" ]; then + _ListDatabasesRemote + if [ $? != 0 ]; then + output_file="" + else + output_file="$RUN_DIR/$PROGRAM._ListDatabasesRemote.$SCRIPT_PID" + fi + fi + + if [ -f "$output_file" ] && [ $CAN_BACKUP_SQL -eq 1 ]; then + OLD_IFS=$IFS + IFS=$' \n' + for line in $(cat "$output_file") + do + db_name=$(echo $line | cut -f1) + db_size=$(echo $line | cut -f2) + + if [ "$DATABASES_ALL" == "yes" ]; then + db_backup=1 + IFS=$PATH_SEPARATOR_CHAR + for j in $DATABASES_ALL_EXCLUDE_LIST + do + if [ "$db_name" == "$j" ]; then + db_backup=0 + fi + done + IFS=$' \n' + else + db_backup=0 + IFS=$PATH_SEPARATOR_CHAR + for j in $DATABASES_LIST + do + if [ "$db_name" == "$j" ]; then + db_backup=1 + fi + done + IFS=$' \n' + fi + + if [ $db_backup -eq 1 ]; then + if [ "$SQL_BACKUP_TASKS" != "" ]; then + SQL_BACKUP_TASKS="$SQL_BACKUP_TASKS $db_name" + else + SQL_BACKUP_TASKS="$db_name" + fi + TOTAL_DATABASES_SIZE=$((TOTAL_DATABASES_SIZE+$db_size)) + else + SQL_EXCLUDED_TASKS="$SQL_EXCLUDED_TASKS $db_name" + fi + done + IFS=$OLD_IFS + + Logger "Database backup list: $SQL_BACKUP_TASKS" "DEBUG" + Logger "Database exclude list: $SQL_EXCLUDED_TASKS" "DEBUG" + else + Logger "Will not execute database backup." "ERROR" + CAN_BACKUP_SQL=0 + fi +} + +function _ListRecursiveBackupDirectoriesLocal { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + OLD_IFS=$IFS + IFS=$PATH_SEPARATOR_CHAR + for directory in $RECURSIVE_DIRECTORY_LIST + do + cmd="$COMMAND_SUDO $FIND_CMD -L $directory/ -mindepth 1 -maxdepth 1 -type d >> $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + if [ $? != 0 ]; then + Logger "Could not enumerate directories in [$directory]." "ERROR" + if [ -f $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + if [ -f $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + retval=1 + else + retval=0 + fi + done + IFS=$OLD_IFS + return $retval +} + +function _ListRecursiveBackupDirectoriesRemote { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + OLD_IFS=$IFS + IFS=$PATH_SEPARATOR_CHAR + for directory in $RECURSIVE_DIRECTORY_LIST + do + cmd=$SSH_CMD' "'$COMMAND_SUDO' '$FIND_CMD' -L '$directory'/ -mindepth 1 -maxdepth 1 -type d" >> '$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + if [ $? != 0 ]; then + Logger "Could not enumerate directories in [$directory]." "ERROR" + if [ -f $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + if [ -f $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + retval=1 + else + retval=0 + fi + done + IFS=$OLD_IFS + return $retval +} + +function ListRecursiveBackupDirectories { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + Logger "Listing directories to backup." "NOTICE" + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]; then + _ListRecursiveBackupDirectoriesLocal + if [ $? != 0 ]; then + output_file="" + else + output_file="$RUN_DIR/$PROGRAM._ListRecursiveBackupDirectoriesLocal.$SCRIPT_PID" + fi + elif [ "$BACKUP_TYPE" == "pull" ]; then + _ListRecursiveBackupDirectoriesRemote + if [ $? != 0 ]; then + output_file="" + else + output_file="$RUN_DIR/$PROGRAM._ListRecursiveBackupDirectoriesRemote.$SCRIPT_PID" + fi + fi + + if [ -f "$output_file" ]; then + OLD_IFS=$IFS + IFS=$' \n' + for line in $(cat "$output_file") + do + file_exclude=0 + IFS=$PATH_SEPARATOR_CHAR + for k in $RECURSIVE_EXCLUDE_LIST + do + if [ "$k" == "$line" ]; then + file_exclude=1 + fi + done + IFS=$' \n' + + if [ $file_exclude -eq 0 ]; then + if [ "$FILE_RECURSIVE_BACKUP_TASKS" == "" ]; then + FILE_RECURSIVE_BACKUP_TASKS="$line" + FILE_SIZE_LIST="$(EscapeSpaces $line)" + else + FILE_RECURSIVE_BACKUP_TASKS="$FILE_RECURSIVE_BACKUP_TASKS$PATH_SEPARATOR_CHAR$line" + FILE_SIZE_LIST="$FILE_SIZE_LIST $(EscapeSpaces $line)" + fi + else + FILE_RECURSIVE_EXCLUDED_TASKS="$FILE_RECURSIVE_EXCLUDED_TASKS$PATH_SEPARATOR_CHAR$line" + fi + done + IFS=$OLD_IFS + fi + + OLD_IFS=$IFS + IFS=$PATH_SEPARATOR_CHAR + for directory in $DIRECTORY_LIST + do + FILE_SIZE_LIST="$FILE_SIZE_LIST $(EscapeSpaces $line)" + if [ "$FILE_BACKUP_TASKS" == "" ]; then + FILE_BACKUP_TASKS="$directory" + else + FILE_BACKUP_TASKS="$FILE_BACKUP_TASKS$PATH_SEPARATOR_CHAR$directory" + fi + done + IFS=$OLD_IFS +} + +function _GetDirectoriesSizeLocal { + local dir_list="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + cmd='echo "'$dir_list'" | xargs '$COMMAND_SUDO' du -cs | tail -n1 | cut -f1 > '$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + # $cmd will return 0 even if some errors found, so we need to check if there is an error output + if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID ]; then + Logger "Could not get files size for some or all directories." "ERROR" + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + else + Logger "File size fetched successfully." "NOTICE" + fi + + if [ -s "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + TOTAL_FILES_SIZE="$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" + else + TOTAL_FILES_SIZE=-1 + fi +} + +function _GetDirectoriesSizeRemote { + local dir_list="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + # Error output is different from stdout because not all files in list may fail at once + cmd=$SSH_CMD' "echo '$dir_list' | xargs '$COMMAND_SUDO' du -cs | tail -n1 | cut -f1" > '$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + # $cmd will return 0 even if some errors found, so we need to check if there is an error output + if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID ]; then + Logger "Could not get files size for some or all directories." "ERROR" + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + else + Logger "File size fetched successfully." "NOTICE" + fi + if [ -s "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + TOTAL_FILES_SIZE="$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" + else + TOTAL_FILES_SIZE=-1 + fi +} + +function GetDirectoriesSize { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + Logger "Getting files size" "NOTICE" + + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]; then + if [ "$FILE_BACKUP" != "no" ]; then + _GetDirectoriesSizeLocal "$FILE_SIZE_LIST" + fi + elif [ "$BACKUP_TYPE" == "pull" ]; then + if [ "$FILE_BACKUP" != "no" ]; then + _GetDirectoriesSizeRemote "$FILE_SIZE_LIST" + fi + fi +} + +function _CreateStorageDirsLocal { + local dir_to_create="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ ! -d "$dir_to_create" ]; then + $COMMAND_SUDO mkdir -p "$dir_to_create" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 + if [ $? != 0 ]; then + Logger "Cannot create directory [$dir_to_create]" "CRITICAL" + if [ -f $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID ]; then + Logger "Command output: $(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + return 1 + fi + fi +} + +function _CreateStorageDirsRemote { + local dir_to_create="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + cmd=$SSH_CMD' "if ! [ -d \"'$dir_to_create'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$dir_to_create'\"; fi" > '$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID' 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 1800 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot create remote directory [$dir_to_create]." "CRITICAL" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + return 1 + fi +} + +function CreateStorageDirectories { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "pull" ]; then + if [ "SQL_BACKUP" != "no" ]; then + _CreateStorageDirsLocal "$SQL_STORAGE" + if [ $? != 0 ]; then + CAN_BACKUP_SQL=0 + fi + fi + if [ "FILE_BACKUP" != "no" ]; then + _CreateStorageDirsLocal "$FILE_STORAGE" + if [ $? != 0 ]; then + CAN_BACKUP_FILES=0 + fi + fi + elif [ "$BACKUP_TYPE" == "push" ]; then + if [ "SQL_BACKUP" != "no" ]; then + _CreateStorageDirsRemote "$SQL_STORAGE" + if [ $? != 0 ]; then + CAN_BACKUP_SQL=0 + fi + fi + if [ "$FILE_BACKUP" != "no" ]; then + _CreateStorageDirsRemote "$FILE_STORAGE" + if [ $? != 0 ]; then + CAN_BACKUP_FILES=0 + fi + fi + fi +} + +function GetDiskSpaceLocal { + # GLOBAL VARIABLE DISK_SPACE to pass variable to parent function + # GLOBAL VARIABLE DRIVE to pass variable to parent function + local path_to_check="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ -w "$path_to_check" ]; then + # Not elegant solution to make df silent on errors + $COMMAND_SUDO df -P "$path_to_check" > "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" 2>&1 + if [ $? != 0 ]; then + DISK_SPACE=0 + Logger "Cannot get disk space in [$path_to_check] on local system." "ERROR" + Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + else + DISK_SPACE=$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID | tail -1 | awk '{print $4}') + DRIVE=$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID | tail -1 | awk '{print $1}') + fi + else + Logger "Storage path [$path_to_check] does not exist or cannot write to it." "CRITICAL" + return 1 + fi +} + +function GetDiskSpaceRemote { + # USE GLOBAL VARIABLE DISK_SPACE to pass variable to parent function + local path_to_check="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + cmd=$SSH_CMD' "if [ -w \"'$path_to_check'\" ]; then '$COMMAND_SUDO' df -P \"'$path_to_check'\"; else exit 1; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + if [ $? != 0 ]; then + DISK_SPACE=0 + Logger "Cannot get disk space in [$path_to_check] on remote system." "ERROR" + Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + return 1 + else + DISK_SPACE=$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID | tail -1 | awk '{print $4}') + DRIVE=$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID | tail -1 | awk '{print $1}') + fi +} + +function CheckDiskSpace { + # USE OF GLOBAL VARIABLES TOTAL_DATABASES_SIZE, TOTAL_FILES_SIZE, BACKUP_SIZE_MINIMUM, STORAGE_WARN_SIZE, STORAGE_SPACE + + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "pull" ]; then + if [ "$SQL_BACKUP" != "no" ]; then + GetDiskSpaceLocal "$SQL_STORAGE" + if [ $? != 0 ]; then + SQL_DISK_SPACE=0 + CAN_BACKUP_SQL=0 + else + SQL_DISK_SPACE=$DISK_SPACE + SQL_DRIVE=$DRIVE + fi + fi + if [ "$FILE_BACKUP" != "no" ]; then + GetDiskSpaceLocal "$FILE_STORAGE" + if [ $? != 0 ]; then + FILE_DISK_SPACE=0 + CAN_BACKUP_FILES=0 + else + FILE_DISK_SPACE=$DISK_SPACE + FILE_DRIVE=$DRIVE + fi + fi + elif [ "$BACKUP_TYPE" == "push" ]; then + if [ "$SQL_BACKUP" != "no" ]; then + GetDiskSpaceRemote "$SQL_STORAGE" + if [ $? != 0 ]; then + SQL_DISK_SPACE=0 + else + SQL_DISK_SPACE=$DISK_SPACE + SQL_DRIVE=$DRIVE + fi + fi + if [ "$FILE_BACKUP" != "no" ]; then + GetDiskSpaceRemote "$FILE_STORAGE" + if [ $? != 0 ]; then + FILE_DISK_SPACE=0 + else + FILE_DISK_SPACE=$DISK_SPACE + FILE_DRIVE=$DRIVE + fi + fi + fi + + if [ "$TOTAL_DATABASES_SIZE" == "" ]; then + TOTAL_DATABASES_SIZE=-1 + fi + if [ "$TOTAL_FILES_SIZE" == "" ]; then + TOTAL_FILES_SIZE=-1 + fi + + if [ "$SQL_BACKUP" != "no" ] && [ $CAN_BACKUP_SQL -eq 1 ]; then + if [ $SQL_DISK_SPACE -eq 0 ]; then + Logger "Storage space in [$SQL_STORAGE] reported to be 0Ko." "WARN" + fi + if [ $SQL_DISK_SPACE -lt $TOTAL_DATABASES_SIZE ]; then + Logger "Disk space in [$SQL_STORAGE] may be insufficient to backup SQL ($SQL_DISK_SPACE Ko available in $SQL_DRIVE) (non compressed databases calculation)." "WARN" + fi + if [ $SQL_DISK_SPACE -lt $SQL_WARN_MIN_SPACE ]; then + Logger "Disk space in [$SQL_STORAGE] is lower than warning value [$SQL_WARN_MIN_SPACE Ko]." "WARN" + fi + Logger "SQL storage Space: $SQL_DISK_SPACE Ko - Databases size: $TOTAL_DATABASES_SIZE Ko" "NOTICE" + fi + + if [ "$FILE_BACKUP" != "no" ] && [ $CAN_BACKUP_FILES -eq 1 ]; then + if [ $FILE_DISK_SPACE -eq 0 ]; then + Logger "Storage space in [$FILE_STORAGE] reported to be 0 Ko." "WARN" + fi + if [ $FILE_DISK_SPACE -lt $TOTAL_FILES_SIZE ]; then + Logger "Disk space in [$FILE_STORAGE] may be insufficient to backup files ($FILE_DISK_SPACE Ko available in $FILE_DRIVE)." "WARN" + fi + if [ $FILE_DISK_SPACE -lt $FILE_WARN_MIN_SPACE ]; then + Logger "Disk space in [$FILE_STORAGE] is lower than warning value [$FILE_WARN_MIN_SPACE Ko]." "WARN" + fi + Logger "File storage space: $FILE_DISK_SPACE Ko - Files size: $TOTAL_FILES_SIZE Ko" "NOTICE" + fi + + if [ $BACKUP_SIZE_MINIMUM -gt $(($TOTAL_DATABASES_SIZE+$TOTAL_FILES_SIZE)) ] && [ "$GET_BACKUP_SIZE" != "no" ]; then + Logger "Backup size is smaller than expected." "WARN" + fi +} + +function _BackupDatabaseLocalToLocal { + local database="${1}" # Database to backup + local export_options="${2}" # export options + + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + local dry_sql_cmd="mysqldump -u $SQL_USER $export_options --database $database $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS > /dev/null 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" + local sql_cmd="mysqldump -u $SQL_USER $export_options --database $database $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS > $SQL_STORAGE/$database.sql$COMPRESSION_EXTENSION 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" + + if [ $_DRYRUN -ne 1 ]; then + Logger "cmd: $sql_cmd" "DEBUG" + eval "$sql_cmd" & + else + Logger "cmd: $dry_sql_cmd" "DEBUG" + eval "$dry_sql_cmd" & + fi + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + local retval=$? + if [ -s "$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + return $retval +} + +function _BackupDatabaseLocalToRemote { + local database="${1}" # Database to backup + local export_options="${2}" # export options + + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + + #TODO-v2.0: cannot catch mysqldump warnings + local dry_sql_cmd="mysqldump -u $SQL_USER $export_options --database $database $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS > /dev/null 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" + local sql_cmd="mysqldump -u $SQL_USER $export_options --database $database $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS | $SSH_CMD '$COMMAND_SUDO tee \"$SQL_STORAGE/$database.sql$COMPRESSION_EXTENSION\" > /dev/null' 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" + + if [ $_DRYRUN -ne 1 ]; then + Logger "cmd: $sql_cmd" "DEBUG" + eval "$sql_cmd" & + else + Logger "cmd: $dry_sql_cmd" "DEBUG" + eval "$dry_sql_cmd" & + fi + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + local retval=$? + if [ -s "$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + return $retval +} + +function _BackupDatabaseRemoteToLocal { + local database="${1}" # Database to backup + local export_options="${2}" # export options + + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + + local dry_sql_cmd=$SSH_CMD' "mysqldump -u '$SQL_USER' '$export_options' --database '$database' '$COMPRESSION_PROGRAM' '$COMPRESSION_OPTIONS'" > /dev/null 2> "'$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID'"' + local sql_cmd=$SSH_CMD' "mysqldump -u '$SQL_USER' '$export_options' --database '$database' '$COMPRESSION_PROGRAM' '$COMPRESSION_OPTIONS'" > "'$SQL_STORAGE/$database.sql$COMPRESSION_EXTENSION'" 2> "'$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID'"' + + if [ $_DRYRUN -ne 1 ]; then + Logger "cmd: $sql_cmd" "DEBUG" + eval "$sql_cmd" & + else + Logger "cmd: $dry_sql_cmd" "DEBUG" + eval "$dry_sql_cmd" & + fi + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + local retval=$? + if [ -s "$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + return $retval +} + +function BackupDatabase { + local database="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + # Hack to prevent warning on table mysql.events, some mysql versions don't support --skip-events, prefer using --ignore-table + if [ "$database" == "mysql" ]; then + local mysql_options='--skip-lock-tables --single-transaction --ignore-table=mysql.event' + else + local mysql_options='--skip-lock-tables --single-transaction' + fi + + if [ "$BACKUP_TYPE" == "local" ]; then + _BackupDatabaseLocalToLocal "$database" "$mysql_options" + elif [ "$BACKUP_TYPE" == "pull" ]; then + _BackupDatabaseRemoteToLocal "$database" "$mysql_options" + elif [ "$BACKUP_TYPE" == "push" ]; then + _BackupDatabaseLocalToRemote "$database" "$mysql_options" + fi + + if [ $? -ne 0 ]; then + Logger "Backup failed." "ERROR" + else + Logger "Backup succeeded." "NOTICE" + fi +} + +function BackupDatabases { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + local database + + OLD_IFS=$IFS + IFS=$' \t\n' + for database in $SQL_BACKUP_TASKS + do + Logger "Backing up database [$database]." "NOTICE" + BackupDatabase $database & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + CheckTotalExecutionTime + done + IFS=$OLD_IFS +} + +function Rsync { + local backup_directory="${1}" # Which directory to backup + local is_recursive="${2}" # Backup only files at toplevel of directory + + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$KEEP_ABSOLUTE_PATHS" == "yes" ]; then + local file_storage_path="$(dirname $FILE_STORAGE$backup_directory)" + else + local file_storage_path="$FILE_STORAGE" + fi + + ## Manage to backup recursive directories lists files only (not recursing into subdirectories) + if [ "$is_recursive" == "no-recurse" ]; then + # Fixes symlinks to directories in target cannot be deleted when backing up root directory without recursion, and excludes subdirectories + RSYNC_NO_RECURSE_ARGS=" -k --exclude=*/*/" + else + RSYNC_NO_RECURSE_ARGS="" + fi + + # Creating subdirectories because rsync cannot handle mkdir -p + if [ ! -d "$file_storage_path/$backup_directory" ]; then + $COMMAND_SUDO mkdir -p "$file_storage_path/$backup_directory" + if [ $? != 0 ]; then + Logger "Cannot create storage path [$file_storage_path/$backup_directory]." "ERROR" + fi + fi + + if [ "$BACKUP_TYPE" == "local" ]; then + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + elif [ "$BACKUP_TYPE" == "pull" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + elif [ "$BACKUP_TYPE" == "push" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$backup_directory\" \"$REMOTE_USER@$REMOTE_HOST:$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + fi + + Logger "cmd: $rsync_cmd" "DEBUG" + eval "$rsync_cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + if [ $? != 0 ]; then + Logger "Failed to backup [$backup_directory] to [$file_storage_path]." "ERROR" + Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + else + Logger "File backup succeed." "NOTICE" + fi +} + +function Duplicity { + local backup_directory="${1}" # Which directory to backup + local is_recursive="${2}" # Backup only files at toplevel of directory + + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + Logger "Encrpytion not supported yet ! No backup done." "CRITICAL" + return 1 + + if [ "$KEEP_ABSOLUTE_PATHS" == "yes" ]; then + local file_storage_path="$(dirname $FILE_STORAGE$backup_directory)" + else + local file_storage_path="$FILE_STORAGE" + fi + + if [ "$BACKUP_TYPE" == "local" ]; then + duplicity_cmd="" + elif [ "$BACKUP_TYPE" == "pull" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + duplicity_cmd="" + elif [ "$BACKUP_TYPE" == "push" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + duplicity_cmd="" + fi + + Logger "cmd: $duplicity_cmd" "DEBUG" + eval "$duplicity_cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + if [ $? != 0 ]; then + Logger "Failed to backup [$backup_directory] to [$file_storage_path]." "ERROR" + Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + else + Logger "File backup succeed." "NOTICE" + fi + +} + +function FilesBackup { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + OLD_IFS=$IFS + IFS=$PATH_SEPARATOR_CHAR + # Backup non recursive directories + for BACKUP_TASK in $FILE_BACKUP_TASKS + do + Logger "Beginning file backup of [$BACKUP_TASK]." "NOTICE" + if [ "$ENCRYPTION" == "yes" ]; then + Duplicity "$BACKUP_TASK" "recurse" + else + Rsync "$BACKUP_TASK" "recurse" + fi + CheckTotalExecutionTime + done + + ## Backup files at root of DIRECTORIES_RECURSE_LIST directories + for BACKUP_TASK in $RECURSIVE_DIRECTORY_LIST + do + Logger "Beginning non recursive file backup of [$BACKUP_TASK]." "NOTICE" + if [ "$ENCRYPTION" == "yes" ]; then + Duplicity "$BACKUP_TASK" "no-recurse" + else + Rsync "$BACKUP_TASK" "no-recurse" + fi + CheckTotalExecutionTime + done + + # Backup sub directories of recursive directories + for BACKUP_TASK in $FILE_RECURSIVE_BACKUP_TASKS + do + Logger "Beginning recursive file backup of [$BACKUP_TASK]." "NOTICE" + if [ "$ENCRYPTION" == "yes" ]; then + Duplicity "$BACKUP_TASK" "recurse" + else + Rsync "$BACKUP_TASK" "recurse" + fi + CheckTotalExecutionTime + done + IFS=$OLD_IFS +} + +function CheckTotalExecutionTime { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + #### Check if max execution time of whole script as been reached + if [ $SECONDS -gt $SOFT_MAX_EXEC_TIME_TOTAL ]; then + Logger "Max soft execution time of the whole backup exceeded while backing up [$BACKUP_TASK]." "ERROR" + WARN_ALERT=1 + SendAlert + if [ $SECONDS -gt $HARD_MAX_EXEC_TIME_TOTAL ] && [ $HARD_MAX_EXEC_TIME_TOTAL -ne 0 ]; then + Logger "Max hard execution time of the whole backup exceeded while backing up [$BACKUP_TASK], stopping backup process." "CRITICAL" + exit 1 + fi + fi +} + +function RsyncExcludePattern { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + # Disable globbing so wildcards from exclusions do not get expanded + set -f + rest="$RSYNC_EXCLUDE_PATTERN" + while [ -n "$rest" ] + do + # Take the string until first occurence until $PATH_SEPARATOR_CHAR + str=${rest%%;*} + # Handle the last case + if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ]; then + rest= + else + # Cut everything before the first occurence of $PATH_SEPARATOR_CHAR + rest=${rest#*$PATH_SEPARATOR_CHAR} + fi + + if [ "$RSYNC_EXCLUDE" == "" ]; then + RSYNC_EXCLUDE="--exclude=\"$str\"" + else + RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$str\"" + fi + done + set +f +} + +function RsyncExcludeFrom { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ ! $RSYNC_EXCLUDE_FROM == "" ]; then + ## Check if the exclude list has a full path, and if not, add the config file path if there is one + if [ "$(basename $RSYNC_EXCLUDE_FROM)" == "$RSYNC_EXCLUDE_FROM" ]; then + RSYNC_EXCLUDE_FROM=$(dirname $ConfigFile)/$RSYNC_EXCLUDE_FROM + fi + + if [ -e $RSYNC_EXCLUDE_FROM ]; then + RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\"" + fi + fi +} + +function _RotateBackupsLocal { + local backup_path="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + OLD_IFS=$IFS + IFS=$'\t\n' + for backup in $(ls -I "*.$PROGRAM.*" "$backup_path") + do + copy=$ROTATE_COPIES + while [ $copy -gt 1 ] + do + if [ $copy -eq $ROTATE_COPIES ]; then + cmd="$COMMAND_SUDO rm -rf \"$backup_path/$backup.$PROGRAM.$copy\"" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot delete oldest copy [$backup_path/$backup.$PROGRAM.$copy]." "ERROR" + fi + fi + path="$backup_path/$backup.$PROGRAM.$(($copy-1))" + if [[ -f $path || -d $path ]]; then + cmd="$COMMAND_SUDO mv \"$path\" \"$backup_path/$backup.$PROGRAM.$copy\"" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot move [$path] to [$backup_path/$backup.$PROGRAM.$copy]." "ERROR" + fi + + fi + copy=$(($copy-1)) + done + + # Latest file backup will not be moved if script configured for remote backup so next rsync execution will only do delta copy instead of full one + if [[ $backup == *.sql.* ]]; then + cmd="$COMMAND_SUDO mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + + elif [ "$REMOTE_OPERATION" == "yes" ]; then + cmd="$COMMAND_SUDO cp -R \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot copy [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + + else + cmd="$COMMAND_SUDO mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + fi + done + IFS=$OLD_IFS +} + +function _RotateBackupsRemote { + local backup_path="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG +$SSH_CMD PROGRAM=$PROGRAM REMOTE_OPERATION=$REMOTE_OPERATION _DEBUG=$_DEBUG COMMAND_SUDO=$COMMAND_SUDO ROTATE_COPIES=$ROTATE_COPIES backup_path="$backup_path" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" 2>&1 & + +function _RemoteLogger { + local value="${1}" # What to log + echo -e "$value" +} + +function RemoteLogger { + local value="${1}" # Sentence to log (in double quotes) + local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL + + prefix="REMOTE TIME: $SECONDS - " + + if [ "$level" == "CRITICAL" ]; then + _RemoteLogger "$prefix\e[41m$value\e[0m" + return + elif [ "$level" == "ERROR" ]; then + _RemoteLogger "$prefix\e[91m$value\e[0m" + return + elif [ "$level" == "WARN" ]; then + _RemoteLogger "$prefix\e[93m$value\e[0m" + return + elif [ "$level" == "NOTICE" ]; then + _RemoteLogger "$prefix$value" + return + elif [ "$level" == "DEBUG" ]; then + if [ "$_DEBUG" == "yes" ]; then + _RemoteLogger "$prefix$value" + return + fi + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG + _RemoteLogger "$prefix$value" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG + else + _RemoteLogger "\e[41mLogger function called without proper loglevel.\e[0m" + _RemoteLogger "$prefix$value" + fi +} + +function _RotateBackupsRemoteSSH { + OLD_IFS=$IFS + IFS=$' \t\n' + for backup in $(ls -I "*.$PROGRAM.*" "$backup_path") + do + copy=$ROTATE_COPIES + while [ $copy -gt 1 ] + do + if [ $copy -eq $ROTATE_COPIES ]; then + cmd="$COMMAND_SUDO rm -rf \"$backup_path/$backup.$PROGRAM.$copy\"" + RemoteLogger "cmd: $cmd" "DEBUG" + eval "$cmd" + if [ $? != 0 ]; then + RemoteLogger "Cannot delete oldest copy [$backup_path/$backup.$PROGRAM.$copy]." "ERROR" + fi + fi + path="$backup_path/$backup.$PROGRAM.$(($copy-1))" + if [[ -f $path || -d $path ]]; then + cmd="$COMMAND_SUDO mv \"$path\" \"$backup_path/$backup.$PROGRAM.$copy\"" + RemoteLogger "cmd: $cmd" "DEBUG" + eval "$cmd" + if [ $? != 0 ]; then + RemoteLogger "Cannot move [$path] to [$backup_path/$backup.$PROGRAM.$copy]." "ERROR" + fi + + fi + copy=$(($copy-1)) + done + + # Latest file backup will not be moved if script configured for remote backup so next rsync execution will only do delta copy instead of full one + if [[ $backup == *.sql.* ]]; then + cmd="$COMMAND_SUDO mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + RemoteLogger "cmd: $cmd" "DEBUG" + eval "$cmd" + if [ $? != 0 ]; then + RemoteLogger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + + elif [ "$REMOTE_OPERATION" == "yes" ]; then + cmd="$COMMAND_SUDO cp -R \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + RemoteLogger "cmd: $cmd" "DEBUG" + eval "$cmd" + if [ $? != 0 ]; then + RemoteLogger "Cannot copy [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + + else + cmd="$COMMAND_SUDO mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + RemoteLogger "cmd: $cmd" "DEBUG" + eval "$cmd" + if [ $? != 0 ]; then + RemoteLogger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + fi + done + IFS=$OLD_IFS +} + + _RotateBackupsRemoteSSH + +ENDSSH + + WaitForTaskCompletion $! 1800 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Could not rotate backups in [$backup_path]." "ERROR" + Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + else + Logger "Remote rotation succeed." "NOTICE" + fi ## Need to add a trivial sleep time to give ssh time to log to local file + #sleep 5 + + +} + +function RotateBackups { + local backup_path="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + Logger "Rotating backups." "NOTICE" + + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "pull" ]; then + _RotateBackupsLocal "$backup_path" + elif [ "$BACKUP_TYPE" == "push" ]; then + _RotateBackupsRemote "$backup_path" + fi +} + +function Init { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + trap TrapStop SIGINT SIGQUIT SIGKILL SIGTERM SIGHUP + trap TrapQuit EXIT + + ## Test if target dir is a ssh uri, and if yes, break it down it its values + if [ "${REMOTE_SYSTEM_URI:0:6}" == "ssh://" ] && [ "$BACKUP_TYPE" != "local" ]; then + REMOTE_OPERATION="yes" + + # remove leadng 'ssh://' + uri=${REMOTE_SYSTEM_URI#ssh://*} + if [[ "$uri" == *"@"* ]]; then + # remove everything after '@' + REMOTE_USER=${uri%@*} + else + REMOTE_USER=$LOCAL_USER + fi + + if [ "$SSH_RSA_PRIVATE_KEY" == "" ]; then + SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa + fi + + # remove everything before '@' + _hosturiandpath=${uri#*@} + # remove everything after first '/' + _hosturi=${_hosturiandpath%%/*} + if [[ "$_hosturi" == *":"* ]]; then + REMOTE_PORT=${_hosturi##*:} + else + REMOTE_PORT=22 + fi + REMOTE_HOST=${_hosturi%%:*} + fi + + ## Add update to default RSYNC_ARGS + RSYNC_ARGS=$RSYNC_ARGS"u" + + if [ $_VERBOSE -eq 1 ]; then + RSYNC_ARGS=$RSYNC_ARGS"i" + fi + + if [ "$PARTIAL" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" + RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$PARTIAL_DIR\"" + fi + + if [ "$DELETE_VANISHED_FILES" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --delete" + fi + + if [ "$DELTA_COPIES" != "no" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file" + else + RSYNC_ARGS=$RSYNC_ARGS" --whole-file" + fi + + if [ $stats -eq 1 ]; then + RSYNC_ARGS=$RSYNC_ARGS" --stats" + fi + + ## Fix for symlink to directories on target cannot get updated + RSYNC_ARGS=$RSYNC_ARGS" --force" +} + +function Main { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$SQL_BACKUP" != "no" ] && [ $CAN_BACKUP_SQL -eq 1 ]; then + ListDatabases + fi + if [ "$FILE_BACKUP" != "no" ] && [ $CAN_BACKUP_FILES -eq 1 ]; then + ListRecursiveBackupDirectories + if [ "$GET_BACKUP_SIZE" != "no" ]; then + GetDirectoriesSize + else + TOTAL_FILE_SIZE=0 + fi + fi + + if [ "$CREATE_DIRS" != "no" ]; then + CreateStorageDirectories + fi + CheckDiskSpace + + # Actual backup process + if [ "$SQL_BACKUP" != "no" ] && [ $CAN_BACKUP_SQL -eq 1 ]; then + if [ $_DRYRUN -ne 1 ] && [ "$ROTATE_SQL_BACKUPS" == "yes" ]; then + RotateBackups "$SQL_STORAGE" + fi + BackupDatabases + fi + + if [ "$FILE_BACKUP" != "no" ] && [ $CAN_BACKUP_FILES -eq 1 ]; then + if [ $_DRYRUN -ne 1 ] && [ "$ROTATE_FILE_BACKUPS" == "yes" ]; then + RotateBackups "$FILE_STORAGE" + fi + ## Add Rsync exclude patterns + RsyncExcludePattern + ## Add Rsync exclude from file + RsyncExcludeFrom + FilesBackup + fi +} + +function Usage { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + + if [ "$IS_STABLE" != "yes" ]; then + echo -e "\e[93mThis is an unstable dev build. Please use with caution.\e[0m" + fi + + echo "$PROGRAM $PROGRAM_VERSION $PROGRAM_BUILD" + echo "$AUTHOR" + echo "$CONTACT" + echo "" + echo "usage: obackup.sh /path/to/backup.conf [OPTIONS]" + echo "" + echo "OPTIONS:" + echo "--dry: will run obackup without actually doing anything, just testing" + echo "--silent: will run obackup without any output to stdout, usefull for cron backups" + echo "--verbose: adds command outputs" + echo "--stats Adds rsync transfer statistics to verbose output" + echo "--partial Allows rsync to keep partial downloads that can be resumed later (experimental)" + echo "--no-maxtime disables any soft and hard execution time checks" + echo "--delete Deletes files on destination that vanished on source" + echo "--dontgetsize Does not try to evaluate backup size" + exit 128 +} + +# Command line argument flags +_DRYRUN=0 +_SILENT=0 +no_maxtime=0 +dontgetsize=0 +stats=0 +PARTIAL=0 + +function GetCommandlineArguments { + if [ $# -eq 0 ]; then + Usage + fi + + for i in "$@" + do + case $i in + --dry) + _DRYRUN=1 + ;; + --silent) + _SILENT=1 + ;; + --verbose) + _VERBOSE=1 + ;; + --stats) + stats=1 + ;; + --partial) + PARTIAL="yes" + ;; + --no-maxtime) + no_maxtime=1 + ;; + --delete) + DELETE_VANISHED_FILES="yes" + ;; + --dontgetsize) + GET_BACKUP_SIZE="no" + ;; + --help|-h|--version|-v) + Usage + ;; + esac + done +} + +GetCommandlineArguments "$@" +CheckEnvironment +LoadConfigFile "$1" +if [ "$LOGFILE" == "" ]; then + if [ -w /var/log ]; then + LOG_FILE=/var/log/$PROGRAM.$INSTANCE_ID.log + else + LOG_FILE=./$PROGRAM.$INSTANCE_ID.log + fi +else + LOG_FILE="$LOGFILE" +fi + +if [ "$IS_STABLE" != "yes" ]; then + Logger "This is an unstable dev build. Please use with caution." "WARN" +fi + + +GetLocalOS +InitLocalOSSettings +PreInit +Init +PostInit +CheckCurrentConfig +if [ "$REMOTE_OPERATION" == "yes" ]; then + GetRemoteOS + InitRemoteOSSettings +fi +DATE=$(date) +Logger "--------------------------------------------------------------------" "NOTICE" +Logger "$DRY_WARNING $DATE - $PROGRAM v$PROGRAM_VERSION $BACKUP_TYPE script begin." "NOTICE" +Logger "--------------------------------------------------------------------" "NOTICE" +Logger "Backup instance [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE" + +if [ $no_maxtime -eq 1 ]; then + SOFT_MAX_EXEC_TIME_DB_TASK=0 + SOFT_MAX_EXEC_TIME_FILE_TASK=0 + HARD_MAX_EXEC_TIME_DB_TASK=0 + HARD_MAX_EXEC_TIME_FILE_TASK=0 + HARD_MAX_EXEC_TIME_TOTAL=0 +fi + +RunBeforeHook +Main +RunAfterHook diff --git a/dev/merge.sh b/dev/merge.sh new file mode 100755 index 0000000..2f96510 --- /dev/null +++ b/dev/merge.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +## Merges ofunctions.sh and n_osync.sh into osync.sh + +PROGRAM=obackup +FUNC_PATH=/home/git/common + +PARANOIA_DEBUG_LINE="__WITH_PARANOIA_DEBUG" +PARANOIA_DEBUG_BEGIN="#__BEGIN_WITH_PARANOIA_DEBUG" +PARANOIA_DEBUG_END="#__END_WITH_PARANOIA_DEBUG" + +function Unexpand { + unexpand n_$PROGRAM.sh > tmp_$PROGRAM.sh +} + +function Merge { + + sed "/source \"\/home\/git\/common\/ofunctions.sh\"/r /home/git/common/ofunctions.sh" tmp_$PROGRAM.sh | grep -v 'source "/home/git/common/ofunctions.sh"' > debug_$PROGRAM.sh + chmod +x debug_$PROGRAM.sh +} + +function CleanDebug { + +# sed explanation +#/pattern1/{ # if pattern1 is found +# p # print it +# :a # loop +# N # and accumulate lines +# /pattern2/!ba # until pattern2 is found +# s/.*\n// # delete the part before pattern2 +#} +#p + + sed -n '/'$PARANOIA_DEBUG_BEGIN'/{p; :a; N; /'$PARANOIA_DEBUG_END'/!ba; s/.*\n//}; p' debug_$PROGRAM.sh | grep -v "$PARANOIA_DEBUG_LINE" > ../$PROGRAM.sh + chmod +x ../$PROGRAM.sh +} + +function CopyCommons { + sed "s/\[prgname\]/$PROGRAM/g" /home/git/common/common_install.sh > ../install.sh + sed "s/\[prgname\]/$PROGRAM/g" /home/git/common/common_batch.sh > ../$PROGRAM-batch.sh + chmod +x ../install.sh + chmod +x ../obackup-batch.sh +} + +Unexpand +Merge +CleanDebug +rm -f tmp_$PROGRAM.sh +CopyCommons diff --git a/dev/n_obackup.sh b/dev/n_obackup.sh new file mode 100755 index 0000000..59615ed --- /dev/null +++ b/dev/n_obackup.sh @@ -0,0 +1,1352 @@ +#!/usr/bin/env bash + +###### Remote push/pull (or local) backup script for files & databases +###### (L) 2013-2015 by Orsiris "Ozy" de Jong (www.netpower.fr) +PROGRAM="obackup" +AUTHOR="(L) 2013-2015 by Orsiris de Jong" +CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" +PROGRAM_VERSION=2.0-pre +PROGRAM_BUILD=2015111107 +IS_STABLE=no + +source "/home/git/common/ofunctions.sh" + +## Working directory for partial downloads +PARTIAL_DIR=".obackup_workdir_partial" + +# List of runtime created global variables +# $SQL_DISK_SPACE, disk space available on target for sql backups +# $FILE_DISK_SPACE, disk space available on target for file backups +# $SQL_BACKUP_TASKS, list of all databases to backup, space separated +# $SQL_EXCLUDED_TASKS, list of all database to exclude from backup, space separated +# $FILE_BACKUP_TASKS list of directories to backup, found in config file +# $FILE_RECURSIVE_BACKUP_TASKS, list of directories to backup, computed from config file recursive list +# $FILE_RECURSIVE_EXCLUDED_TASKS, list of all directories excluded from recursive list +# $FILE_SIZE_LIST, list of all directories to include in GetDirectoriesSize + +CAN_BACKUP_SQL=1 +CAN_BACKUP_FILES=1 + +function TrapStop { + Logger " /!\ WARNING: Manual exit of backup script. Backups may be in inconsistent state." "WARN" + exit 1 +} + +function TrapQuit { + if [ $ERROR_ALERT -ne 0 ]; then + SendAlert + CleanUp + Logger "Backup script finished with errors." "ERROR" + elif [ $WARN_ALERT -ne 0 ]; then + SendAlert + CleanUp + Logger "Backup script finished with warnings." "WARN" + else + CleanUp + Logger "Backup script finshed." "NOTICE" + fi + + KillChilds $$ > /dev/null 2>&1 +} + +function CheckEnvironment { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$REMOTE_OPERATION" == "yes" ]; then + if ! type ssh > /dev/null 2>&1 ; then + Logger "ssh not present. Cannot start backup." "CRITICAL" + exit 1 + fi + + if [ "$SQL_BACKUP" != "no" ]; then + if ! type mysqldump > /dev/null 2>&1 ; then + Logger "mysqldump not present. Cannot backup SQL." "CRITICAL" + CAN_BACKUP_SQL=0 + fi + if ! type mysql > /dev/null 2>&1 ; then + Logger "mysql not present. Cannot backup SQL." "CRITICAL" + CAN_BACKUP_SQL=0 + fi + fi + fi + + if [ "$FILE_BACKUP" != "no" ]; then + if [ "$ENCRYPTION" == "yes" ]; then + if ! type duplicity > /dev/null 2>&1 ; then + Logger "duplicity not present. Cannot backup encrypted files." "CRITICAL" + CAN_BACKUP_FILES=0 + fi + else + if ! type rsync > /dev/null 2>&1 ; then + Logger "rsync not present. Cannot backup files." "CRITICAL" + CAN_BACKUP_FILES=0 + fi + fi + fi +} + +function CheckCurrentConfig { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$INSTANCE_ID" == "" ]; then + Logger "No INSTANCE_ID defined in config file." "CRITICAL" + exit 1 + fi + + # Check all variables that should contain "yes" or "no" + declare -a yes_no_vars=(SQL_BACKUP FILE_BACKUP ENCRYPTION CREATE_DIRS KEEP_ABSOLUTE_PATHS GET_BACKUP_SIZE SUDO_EXEC SSH_COMPRESSION REMOTE_HOST_PING DATABASES_ALL PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS RSYNC_COMPRESS PARTIAL DELETE_VANISHED_FILES DELTA_COPIES ROTATE_SQL_BACKUPS ROTATE_FILE_BACKUPS STOP_ON_CMD_ERROR) + for i in ${yes_no_vars[@]}; do + test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file.\" \"CRITICAL\"; exit 1; fi" + eval "$test" + done + + if [ "$BACKUP_TYPE" != "local" ] && [ "$BACKUP_TYPE" != "pull" ] && [ "$BACKUP_TYPE" != "push" ]; then + Logger "Bogus BACKUP_TYPE value in config file." "CRITICAL" + exit 1 + fi + + # Check all variables that should contain a numerical value >= 0 + declare -a num_vars=(BACKUP_SIZE_MINIMUM BANDWIDTH SQL_WARN_MIN_SPACE FILE_WARN_MIN_SPACE SOFT_MAX_EXEC_TIME_DB_TASK HARD_MAX_EXEC_TIME_DB_TASK COMPRESSION_LEVEL SOFT_MAX_EXEC_TIME_FILE_TASK HARD_MAX_EXEC_TIME_FILE_TASK SOFT_MAX_EXEC_TIME_TOTAL HARD_MAX_EXEC_TIME_TOTAL ROTATE_COPIES MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER) + for i in ${num_vars[@]}; do + test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file.\" \"CRITICAL\"; exit 1; fi" + eval "$test" + done + + #TODO-v2.1: Add runtime variable tests (RSYNC_ARGS etc) +} + +function _ListDatabasesLocal { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + sql_cmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + Logger "cmd: $sql_cmd" "DEBUG" + eval "$sql_cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + if [ $? -eq 0 ]; then + Logger "Listing databases succeeded." "NOTICE" + else + Logger "Listing databases failed." "ERROR" + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + return 1 + fi + +} + +function _ListDatabasesRemote { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + sql_cmd="$SSH_CMD \"mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > \"$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID\" 2>&1" + Logger "cmd: $sql_cmd" "DEBUG" + eval "$sql_cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + if [ $? -eq 0 ]; then + Logger "Listing databases succeeded." "NOTICE" + else + Logger "Listing databases failed." "ERROR" + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + return 1 + fi +} + +function ListDatabases { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + local output_file # Return of subfunction + + if [ $CAN_BACKUP_SQL -ne 1 ]; then + Logger "Cannot list databases." "ERROR" + return 1 + fi + + Logger "Listing databases." "NOTICE" + + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]; then + _ListDatabasesLocal + if [ $? != 0 ]; then + output_file="" + else + output_file="$RUN_DIR/$PROGRAM._ListDatabasesLocal.$SCRIPT_PID" + fi + elif [ "$BACKUP_TYPE" == "pull" ]; then + _ListDatabasesRemote + if [ $? != 0 ]; then + output_file="" + else + output_file="$RUN_DIR/$PROGRAM._ListDatabasesRemote.$SCRIPT_PID" + fi + fi + + if [ -f "$output_file" ] && [ $CAN_BACKUP_SQL -eq 1 ]; then + OLD_IFS=$IFS + IFS=$' \n' + for line in $(cat "$output_file") + do + db_name=$(echo $line | cut -f1) + db_size=$(echo $line | cut -f2) + + if [ "$DATABASES_ALL" == "yes" ]; then + db_backup=1 + IFS=$PATH_SEPARATOR_CHAR + for j in $DATABASES_ALL_EXCLUDE_LIST + do + if [ "$db_name" == "$j" ]; then + db_backup=0 + fi + done + IFS=$' \n' + else + db_backup=0 + IFS=$PATH_SEPARATOR_CHAR + for j in $DATABASES_LIST + do + if [ "$db_name" == "$j" ]; then + db_backup=1 + fi + done + IFS=$' \n' + fi + + if [ $db_backup -eq 1 ]; then + if [ "$SQL_BACKUP_TASKS" != "" ]; then + SQL_BACKUP_TASKS="$SQL_BACKUP_TASKS $db_name" + else + SQL_BACKUP_TASKS="$db_name" + fi + TOTAL_DATABASES_SIZE=$((TOTAL_DATABASES_SIZE+$db_size)) + else + SQL_EXCLUDED_TASKS="$SQL_EXCLUDED_TASKS $db_name" + fi + done + IFS=$OLD_IFS + + Logger "Database backup list: $SQL_BACKUP_TASKS" "DEBUG" + Logger "Database exclude list: $SQL_EXCLUDED_TASKS" "DEBUG" + else + Logger "Will not execute database backup." "ERROR" + CAN_BACKUP_SQL=0 + fi +} + +function _ListRecursiveBackupDirectoriesLocal { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + OLD_IFS=$IFS + IFS=$PATH_SEPARATOR_CHAR + for directory in $RECURSIVE_DIRECTORY_LIST + do + cmd="$COMMAND_SUDO $FIND_CMD -L $directory/ -mindepth 1 -maxdepth 1 -type d >> $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + if [ $? != 0 ]; then + Logger "Could not enumerate directories in [$directory]." "ERROR" + if [ -f $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + if [ -f $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + retval=1 + else + retval=0 + fi + done + IFS=$OLD_IFS + return $retval +} + +function _ListRecursiveBackupDirectoriesRemote { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + OLD_IFS=$IFS + IFS=$PATH_SEPARATOR_CHAR + for directory in $RECURSIVE_DIRECTORY_LIST + do + cmd=$SSH_CMD' "'$COMMAND_SUDO' '$FIND_CMD' -L '$directory'/ -mindepth 1 -maxdepth 1 -type d" >> '$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + if [ $? != 0 ]; then + Logger "Could not enumerate directories in [$directory]." "ERROR" + if [ -f $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + if [ -f $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + retval=1 + else + retval=0 + fi + done + IFS=$OLD_IFS + return $retval +} + +function ListRecursiveBackupDirectories { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + Logger "Listing directories to backup." "NOTICE" + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]; then + _ListRecursiveBackupDirectoriesLocal + if [ $? != 0 ]; then + output_file="" + else + output_file="$RUN_DIR/$PROGRAM._ListRecursiveBackupDirectoriesLocal.$SCRIPT_PID" + fi + elif [ "$BACKUP_TYPE" == "pull" ]; then + _ListRecursiveBackupDirectoriesRemote + if [ $? != 0 ]; then + output_file="" + else + output_file="$RUN_DIR/$PROGRAM._ListRecursiveBackupDirectoriesRemote.$SCRIPT_PID" + fi + fi + + if [ -f "$output_file" ]; then + OLD_IFS=$IFS + IFS=$' \n' + for line in $(cat "$output_file") + do + file_exclude=0 + IFS=$PATH_SEPARATOR_CHAR + for k in $RECURSIVE_EXCLUDE_LIST + do + if [ "$k" == "$line" ]; then + file_exclude=1 + fi + done + IFS=$' \n' + + if [ $file_exclude -eq 0 ]; then + if [ "$FILE_RECURSIVE_BACKUP_TASKS" == "" ]; then + FILE_RECURSIVE_BACKUP_TASKS="$line" + FILE_SIZE_LIST="$(EscapeSpaces $line)" + else + FILE_RECURSIVE_BACKUP_TASKS="$FILE_RECURSIVE_BACKUP_TASKS$PATH_SEPARATOR_CHAR$line" + FILE_SIZE_LIST="$FILE_SIZE_LIST $(EscapeSpaces $line)" + fi + else + FILE_RECURSIVE_EXCLUDED_TASKS="$FILE_RECURSIVE_EXCLUDED_TASKS$PATH_SEPARATOR_CHAR$line" + fi + done + IFS=$OLD_IFS + fi + + OLD_IFS=$IFS + IFS=$PATH_SEPARATOR_CHAR + for directory in $DIRECTORY_LIST + do + FILE_SIZE_LIST="$FILE_SIZE_LIST $(EscapeSpaces $line)" + if [ "$FILE_BACKUP_TASKS" == "" ]; then + FILE_BACKUP_TASKS="$directory" + else + FILE_BACKUP_TASKS="$FILE_BACKUP_TASKS$PATH_SEPARATOR_CHAR$directory" + fi + done + IFS=$OLD_IFS +} + +function _GetDirectoriesSizeLocal { + local dir_list="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + cmd='echo "'$dir_list'" | xargs '$COMMAND_SUDO' du -cs | tail -n1 | cut -f1 > '$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + # $cmd will return 0 even if some errors found, so we need to check if there is an error output + if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID ]; then + Logger "Could not get files size for some or all directories." "ERROR" + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + else + Logger "File size fetched successfully." "NOTICE" + fi + + if [ -s "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + TOTAL_FILES_SIZE="$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" + else + TOTAL_FILES_SIZE=-1 + fi +} + +function _GetDirectoriesSizeRemote { + local dir_list="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + # Error output is different from stdout because not all files in list may fail at once + cmd=$SSH_CMD' "echo '$dir_list' | xargs '$COMMAND_SUDO' du -cs | tail -n1 | cut -f1" > '$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + # $cmd will return 0 even if some errors found, so we need to check if there is an error output + if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID ]; then + Logger "Could not get files size for some or all directories." "ERROR" + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + else + Logger "File size fetched successfully." "NOTICE" + fi + if [ -s "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + TOTAL_FILES_SIZE="$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" + else + TOTAL_FILES_SIZE=-1 + fi +} + +function GetDirectoriesSize { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + Logger "Getting files size" "NOTICE" + + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]; then + if [ "$FILE_BACKUP" != "no" ]; then + _GetDirectoriesSizeLocal "$FILE_SIZE_LIST" + fi + elif [ "$BACKUP_TYPE" == "pull" ]; then + if [ "$FILE_BACKUP" != "no" ]; then + _GetDirectoriesSizeRemote "$FILE_SIZE_LIST" + fi + fi +} + +function _CreateStorageDirsLocal { + local dir_to_create="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ ! -d "$dir_to_create" ]; then + $COMMAND_SUDO mkdir -p "$dir_to_create" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 + if [ $? != 0 ]; then + Logger "Cannot create directory [$dir_to_create]" "CRITICAL" + if [ -f $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID ]; then + Logger "Command output: $(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + return 1 + fi + fi +} + +function _CreateStorageDirsRemote { + local dir_to_create="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + cmd=$SSH_CMD' "if ! [ -d \"'$dir_to_create'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$dir_to_create'\"; fi" > '$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID' 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 1800 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot create remote directory [$dir_to_create]." "CRITICAL" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + return 1 + fi +} + +function CreateStorageDirectories { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "pull" ]; then + if [ "SQL_BACKUP" != "no" ]; then + _CreateStorageDirsLocal "$SQL_STORAGE" + if [ $? != 0 ]; then + CAN_BACKUP_SQL=0 + fi + fi + if [ "FILE_BACKUP" != "no" ]; then + _CreateStorageDirsLocal "$FILE_STORAGE" + if [ $? != 0 ]; then + CAN_BACKUP_FILES=0 + fi + fi + elif [ "$BACKUP_TYPE" == "push" ]; then + if [ "SQL_BACKUP" != "no" ]; then + _CreateStorageDirsRemote "$SQL_STORAGE" + if [ $? != 0 ]; then + CAN_BACKUP_SQL=0 + fi + fi + if [ "$FILE_BACKUP" != "no" ]; then + _CreateStorageDirsRemote "$FILE_STORAGE" + if [ $? != 0 ]; then + CAN_BACKUP_FILES=0 + fi + fi + fi +} + +function GetDiskSpaceLocal { + # GLOBAL VARIABLE DISK_SPACE to pass variable to parent function + # GLOBAL VARIABLE DRIVE to pass variable to parent function + local path_to_check="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ -w "$path_to_check" ]; then + # Not elegant solution to make df silent on errors + $COMMAND_SUDO df -P "$path_to_check" > "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" 2>&1 + if [ $? != 0 ]; then + DISK_SPACE=0 + Logger "Cannot get disk space in [$path_to_check] on local system." "ERROR" + Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + else + DISK_SPACE=$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID | tail -1 | awk '{print $4}') + DRIVE=$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID | tail -1 | awk '{print $1}') + fi + else + Logger "Storage path [$path_to_check] does not exist or cannot write to it." "CRITICAL" + return 1 + fi +} + +function GetDiskSpaceRemote { + # USE GLOBAL VARIABLE DISK_SPACE to pass variable to parent function + local path_to_check="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + cmd=$SSH_CMD' "if [ -w \"'$path_to_check'\" ]; then '$COMMAND_SUDO' df -P \"'$path_to_check'\"; else exit 1; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + if [ $? != 0 ]; then + DISK_SPACE=0 + Logger "Cannot get disk space in [$path_to_check] on remote system." "ERROR" + Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + return 1 + else + DISK_SPACE=$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID | tail -1 | awk '{print $4}') + DRIVE=$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID | tail -1 | awk '{print $1}') + fi +} + +function CheckDiskSpace { + # USE OF GLOBAL VARIABLES TOTAL_DATABASES_SIZE, TOTAL_FILES_SIZE, BACKUP_SIZE_MINIMUM, STORAGE_WARN_SIZE, STORAGE_SPACE + + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "pull" ]; then + if [ "$SQL_BACKUP" != "no" ]; then + GetDiskSpaceLocal "$SQL_STORAGE" + if [ $? != 0 ]; then + SQL_DISK_SPACE=0 + CAN_BACKUP_SQL=0 + else + SQL_DISK_SPACE=$DISK_SPACE + SQL_DRIVE=$DRIVE + fi + fi + if [ "$FILE_BACKUP" != "no" ]; then + GetDiskSpaceLocal "$FILE_STORAGE" + if [ $? != 0 ]; then + FILE_DISK_SPACE=0 + CAN_BACKUP_FILES=0 + else + FILE_DISK_SPACE=$DISK_SPACE + FILE_DRIVE=$DRIVE + fi + fi + elif [ "$BACKUP_TYPE" == "push" ]; then + if [ "$SQL_BACKUP" != "no" ]; then + GetDiskSpaceRemote "$SQL_STORAGE" + if [ $? != 0 ]; then + SQL_DISK_SPACE=0 + else + SQL_DISK_SPACE=$DISK_SPACE + SQL_DRIVE=$DRIVE + fi + fi + if [ "$FILE_BACKUP" != "no" ]; then + GetDiskSpaceRemote "$FILE_STORAGE" + if [ $? != 0 ]; then + FILE_DISK_SPACE=0 + else + FILE_DISK_SPACE=$DISK_SPACE + FILE_DRIVE=$DRIVE + fi + fi + fi + + if [ "$TOTAL_DATABASES_SIZE" == "" ]; then + TOTAL_DATABASES_SIZE=-1 + fi + if [ "$TOTAL_FILES_SIZE" == "" ]; then + TOTAL_FILES_SIZE=-1 + fi + + if [ "$SQL_BACKUP" != "no" ] && [ $CAN_BACKUP_SQL -eq 1 ]; then + if [ $SQL_DISK_SPACE -eq 0 ]; then + Logger "Storage space in [$SQL_STORAGE] reported to be 0Ko." "WARN" + fi + if [ $SQL_DISK_SPACE -lt $TOTAL_DATABASES_SIZE ]; then + Logger "Disk space in [$SQL_STORAGE] may be insufficient to backup SQL ($SQL_DISK_SPACE Ko available in $SQL_DRIVE) (non compressed databases calculation)." "WARN" + fi + if [ $SQL_DISK_SPACE -lt $SQL_WARN_MIN_SPACE ]; then + Logger "Disk space in [$SQL_STORAGE] is lower than warning value [$SQL_WARN_MIN_SPACE Ko]." "WARN" + fi + Logger "SQL storage Space: $SQL_DISK_SPACE Ko - Databases size: $TOTAL_DATABASES_SIZE Ko" "NOTICE" + fi + + if [ "$FILE_BACKUP" != "no" ] && [ $CAN_BACKUP_FILES -eq 1 ]; then + if [ $FILE_DISK_SPACE -eq 0 ]; then + Logger "Storage space in [$FILE_STORAGE] reported to be 0 Ko." "WARN" + fi + if [ $FILE_DISK_SPACE -lt $TOTAL_FILES_SIZE ]; then + Logger "Disk space in [$FILE_STORAGE] may be insufficient to backup files ($FILE_DISK_SPACE Ko available in $FILE_DRIVE)." "WARN" + fi + if [ $FILE_DISK_SPACE -lt $FILE_WARN_MIN_SPACE ]; then + Logger "Disk space in [$FILE_STORAGE] is lower than warning value [$FILE_WARN_MIN_SPACE Ko]." "WARN" + fi + Logger "File storage space: $FILE_DISK_SPACE Ko - Files size: $TOTAL_FILES_SIZE Ko" "NOTICE" + fi + + if [ $BACKUP_SIZE_MINIMUM -gt $(($TOTAL_DATABASES_SIZE+$TOTAL_FILES_SIZE)) ] && [ "$GET_BACKUP_SIZE" != "no" ]; then + Logger "Backup size is smaller than expected." "WARN" + fi +} + +function _BackupDatabaseLocalToLocal { + local database="${1}" # Database to backup + local export_options="${2}" # export options + + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + local dry_sql_cmd="mysqldump -u $SQL_USER $export_options --database $database $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS > /dev/null 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" + local sql_cmd="mysqldump -u $SQL_USER $export_options --database $database $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS > $SQL_STORAGE/$database.sql$COMPRESSION_EXTENSION 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" + + if [ $_DRYRUN -ne 1 ]; then + Logger "cmd: $sql_cmd" "DEBUG" + eval "$sql_cmd" & + else + Logger "cmd: $dry_sql_cmd" "DEBUG" + eval "$dry_sql_cmd" & + fi + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + local retval=$? + if [ -s "$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + return $retval +} + +function _BackupDatabaseLocalToRemote { + local database="${1}" # Database to backup + local export_options="${2}" # export options + + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + + #TODO-v2.0: cannot catch mysqldump warnings + local dry_sql_cmd="mysqldump -u $SQL_USER $export_options --database $database $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS > /dev/null 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" + local sql_cmd="mysqldump -u $SQL_USER $export_options --database $database $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS | $SSH_CMD '$COMMAND_SUDO tee \"$SQL_STORAGE/$database.sql$COMPRESSION_EXTENSION\" > /dev/null' 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" + + if [ $_DRYRUN -ne 1 ]; then + Logger "cmd: $sql_cmd" "DEBUG" + eval "$sql_cmd" & + else + Logger "cmd: $dry_sql_cmd" "DEBUG" + eval "$dry_sql_cmd" & + fi + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + local retval=$? + if [ -s "$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + return $retval +} + +function _BackupDatabaseRemoteToLocal { + local database="${1}" # Database to backup + local export_options="${2}" # export options + + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + + local dry_sql_cmd=$SSH_CMD' "mysqldump -u '$SQL_USER' '$export_options' --database '$database' '$COMPRESSION_PROGRAM' '$COMPRESSION_OPTIONS'" > /dev/null 2> "'$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID'"' + local sql_cmd=$SSH_CMD' "mysqldump -u '$SQL_USER' '$export_options' --database '$database' '$COMPRESSION_PROGRAM' '$COMPRESSION_OPTIONS'" > "'$SQL_STORAGE/$database.sql$COMPRESSION_EXTENSION'" 2> "'$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID'"' + + if [ $_DRYRUN -ne 1 ]; then + Logger "cmd: $sql_cmd" "DEBUG" + eval "$sql_cmd" & + else + Logger "cmd: $dry_sql_cmd" "DEBUG" + eval "$dry_sql_cmd" & + fi + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + local retval=$? + if [ -s "$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + return $retval +} + +function BackupDatabase { + local database="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + # Hack to prevent warning on table mysql.events, some mysql versions don't support --skip-events, prefer using --ignore-table + if [ "$database" == "mysql" ]; then + local mysql_options='--skip-lock-tables --single-transaction --ignore-table=mysql.event' + else + local mysql_options='--skip-lock-tables --single-transaction' + fi + + if [ "$BACKUP_TYPE" == "local" ]; then + _BackupDatabaseLocalToLocal "$database" "$mysql_options" + elif [ "$BACKUP_TYPE" == "pull" ]; then + _BackupDatabaseRemoteToLocal "$database" "$mysql_options" + elif [ "$BACKUP_TYPE" == "push" ]; then + _BackupDatabaseLocalToRemote "$database" "$mysql_options" + fi + + if [ $? -ne 0 ]; then + Logger "Backup failed." "ERROR" + else + Logger "Backup succeeded." "NOTICE" + fi +} + +function BackupDatabases { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + local database + + OLD_IFS=$IFS + IFS=$' \t\n' + for database in $SQL_BACKUP_TASKS + do + Logger "Backing up database [$database]." "NOTICE" + BackupDatabase $database & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + CheckTotalExecutionTime + done + IFS=$OLD_IFS +} + +function Rsync { + local backup_directory="${1}" # Which directory to backup + local is_recursive="${2}" # Backup only files at toplevel of directory + + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$KEEP_ABSOLUTE_PATHS" == "yes" ]; then + local file_storage_path="$(dirname $FILE_STORAGE$backup_directory)" + else + local file_storage_path="$FILE_STORAGE" + fi + + ## Manage to backup recursive directories lists files only (not recursing into subdirectories) + if [ "$is_recursive" == "no-recurse" ]; then + # Fixes symlinks to directories in target cannot be deleted when backing up root directory without recursion, and excludes subdirectories + RSYNC_NO_RECURSE_ARGS=" -k --exclude=*/*/" + else + RSYNC_NO_RECURSE_ARGS="" + fi + + # Creating subdirectories because rsync cannot handle mkdir -p + if [ ! -d "$file_storage_path/$backup_directory" ]; then + $COMMAND_SUDO mkdir -p "$file_storage_path/$backup_directory" + if [ $? != 0 ]; then + Logger "Cannot create storage path [$file_storage_path/$backup_directory]." "ERROR" + fi + fi + + if [ "$BACKUP_TYPE" == "local" ]; then + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + elif [ "$BACKUP_TYPE" == "pull" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + elif [ "$BACKUP_TYPE" == "push" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$backup_directory\" \"$REMOTE_USER@$REMOTE_HOST:$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + fi + + Logger "cmd: $rsync_cmd" "DEBUG" + eval "$rsync_cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + if [ $? != 0 ]; then + Logger "Failed to backup [$backup_directory] to [$file_storage_path]." "ERROR" + Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + else + Logger "File backup succeed." "NOTICE" + fi +} + +function Duplicity { + local backup_directory="${1}" # Which directory to backup + local is_recursive="${2}" # Backup only files at toplevel of directory + + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + Logger "Encrpytion not supported yet ! No backup done." "CRITICAL" + return 1 + + if [ "$KEEP_ABSOLUTE_PATHS" == "yes" ]; then + local file_storage_path="$(dirname $FILE_STORAGE$backup_directory)" + else + local file_storage_path="$FILE_STORAGE" + fi + + if [ "$BACKUP_TYPE" == "local" ]; then + duplicity_cmd="" + elif [ "$BACKUP_TYPE" == "pull" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + duplicity_cmd="" + elif [ "$BACKUP_TYPE" == "push" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + duplicity_cmd="" + fi + + Logger "cmd: $duplicity_cmd" "DEBUG" + eval "$duplicity_cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + if [ $? != 0 ]; then + Logger "Failed to backup [$backup_directory] to [$file_storage_path]." "ERROR" + Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + else + Logger "File backup succeed." "NOTICE" + fi + +} + +function FilesBackup { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + OLD_IFS=$IFS + IFS=$PATH_SEPARATOR_CHAR + # Backup non recursive directories + for BACKUP_TASK in $FILE_BACKUP_TASKS + do + Logger "Beginning file backup of [$BACKUP_TASK]." "NOTICE" + if [ "$ENCRYPTION" == "yes" ]; then + Duplicity "$BACKUP_TASK" "recurse" + else + Rsync "$BACKUP_TASK" "recurse" + fi + CheckTotalExecutionTime + done + + ## Backup files at root of DIRECTORIES_RECURSE_LIST directories + for BACKUP_TASK in $RECURSIVE_DIRECTORY_LIST + do + Logger "Beginning non recursive file backup of [$BACKUP_TASK]." "NOTICE" + if [ "$ENCRYPTION" == "yes" ]; then + Duplicity "$BACKUP_TASK" "no-recurse" + else + Rsync "$BACKUP_TASK" "no-recurse" + fi + CheckTotalExecutionTime + done + + # Backup sub directories of recursive directories + for BACKUP_TASK in $FILE_RECURSIVE_BACKUP_TASKS + do + Logger "Beginning recursive file backup of [$BACKUP_TASK]." "NOTICE" + if [ "$ENCRYPTION" == "yes" ]; then + Duplicity "$BACKUP_TASK" "recurse" + else + Rsync "$BACKUP_TASK" "recurse" + fi + CheckTotalExecutionTime + done + IFS=$OLD_IFS +} + +function CheckTotalExecutionTime { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + #### Check if max execution time of whole script as been reached + if [ $SECONDS -gt $SOFT_MAX_EXEC_TIME_TOTAL ]; then + Logger "Max soft execution time of the whole backup exceeded while backing up [$BACKUP_TASK]." "ERROR" + WARN_ALERT=1 + SendAlert + if [ $SECONDS -gt $HARD_MAX_EXEC_TIME_TOTAL ] && [ $HARD_MAX_EXEC_TIME_TOTAL -ne 0 ]; then + Logger "Max hard execution time of the whole backup exceeded while backing up [$BACKUP_TASK], stopping backup process." "CRITICAL" + exit 1 + fi + fi +} + +function RsyncExcludePattern { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + # Disable globbing so wildcards from exclusions do not get expanded + set -f + rest="$RSYNC_EXCLUDE_PATTERN" + while [ -n "$rest" ] + do + # Take the string until first occurence until $PATH_SEPARATOR_CHAR + str=${rest%%;*} + # Handle the last case + if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ]; then + rest= + else + # Cut everything before the first occurence of $PATH_SEPARATOR_CHAR + rest=${rest#*$PATH_SEPARATOR_CHAR} + fi + + if [ "$RSYNC_EXCLUDE" == "" ]; then + RSYNC_EXCLUDE="--exclude=\"$str\"" + else + RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$str\"" + fi + done + set +f +} + +function RsyncExcludeFrom { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ ! $RSYNC_EXCLUDE_FROM == "" ]; then + ## Check if the exclude list has a full path, and if not, add the config file path if there is one + if [ "$(basename $RSYNC_EXCLUDE_FROM)" == "$RSYNC_EXCLUDE_FROM" ]; then + RSYNC_EXCLUDE_FROM=$(dirname $ConfigFile)/$RSYNC_EXCLUDE_FROM + fi + + if [ -e $RSYNC_EXCLUDE_FROM ]; then + RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\"" + fi + fi +} + +function _RotateBackupsLocal { + local backup_path="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + OLD_IFS=$IFS + IFS=$'\t\n' + for backup in $(ls -I "*.$PROGRAM.*" "$backup_path") + do + copy=$ROTATE_COPIES + while [ $copy -gt 1 ] + do + if [ $copy -eq $ROTATE_COPIES ]; then + cmd="$COMMAND_SUDO rm -rf \"$backup_path/$backup.$PROGRAM.$copy\"" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot delete oldest copy [$backup_path/$backup.$PROGRAM.$copy]." "ERROR" + fi + fi + path="$backup_path/$backup.$PROGRAM.$(($copy-1))" + if [[ -f $path || -d $path ]]; then + cmd="$COMMAND_SUDO mv \"$path\" \"$backup_path/$backup.$PROGRAM.$copy\"" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot move [$path] to [$backup_path/$backup.$PROGRAM.$copy]." "ERROR" + fi + + fi + copy=$(($copy-1)) + done + + # Latest file backup will not be moved if script configured for remote backup so next rsync execution will only do delta copy instead of full one + if [[ $backup == *.sql.* ]]; then + cmd="$COMMAND_SUDO mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + + elif [ "$REMOTE_OPERATION" == "yes" ]; then + cmd="$COMMAND_SUDO cp -R \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot copy [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + + else + cmd="$COMMAND_SUDO mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + fi + done + IFS=$OLD_IFS +} + +function _RotateBackupsRemote { + local backup_path="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG +$SSH_CMD PROGRAM=$PROGRAM REMOTE_OPERATION=$REMOTE_OPERATION _DEBUG=$_DEBUG COMMAND_SUDO=$COMMAND_SUDO ROTATE_COPIES=$ROTATE_COPIES backup_path="$backup_path" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" 2>&1 & + +function _RemoteLogger { + local value="${1}" # What to log + echo -e "$value" +} + +function RemoteLogger { + local value="${1}" # Sentence to log (in double quotes) + local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL + + prefix="REMOTE TIME: $SECONDS - " + + if [ "$level" == "CRITICAL" ]; then + _RemoteLogger "$prefix\e[41m$value\e[0m" + return + elif [ "$level" == "ERROR" ]; then + _RemoteLogger "$prefix\e[91m$value\e[0m" + return + elif [ "$level" == "WARN" ]; then + _RemoteLogger "$prefix\e[93m$value\e[0m" + return + elif [ "$level" == "NOTICE" ]; then + _RemoteLogger "$prefix$value" + return + elif [ "$level" == "DEBUG" ]; then + if [ "$_DEBUG" == "yes" ]; then + _RemoteLogger "$prefix$value" + return + fi + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG + _RemoteLogger "$prefix$value" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG + else + _RemoteLogger "\e[41mLogger function called without proper loglevel.\e[0m" + _RemoteLogger "$prefix$value" + fi +} + +function _RotateBackupsRemoteSSH { + OLD_IFS=$IFS + IFS=$' \t\n' + for backup in $(ls -I "*.$PROGRAM.*" "$backup_path") + do + copy=$ROTATE_COPIES + while [ $copy -gt 1 ] + do + if [ $copy -eq $ROTATE_COPIES ]; then + cmd="$COMMAND_SUDO rm -rf \"$backup_path/$backup.$PROGRAM.$copy\"" + RemoteLogger "cmd: $cmd" "DEBUG" + eval "$cmd" + if [ $? != 0 ]; then + RemoteLogger "Cannot delete oldest copy [$backup_path/$backup.$PROGRAM.$copy]." "ERROR" + fi + fi + path="$backup_path/$backup.$PROGRAM.$(($copy-1))" + if [[ -f $path || -d $path ]]; then + cmd="$COMMAND_SUDO mv \"$path\" \"$backup_path/$backup.$PROGRAM.$copy\"" + RemoteLogger "cmd: $cmd" "DEBUG" + eval "$cmd" + if [ $? != 0 ]; then + RemoteLogger "Cannot move [$path] to [$backup_path/$backup.$PROGRAM.$copy]." "ERROR" + fi + + fi + copy=$(($copy-1)) + done + + # Latest file backup will not be moved if script configured for remote backup so next rsync execution will only do delta copy instead of full one + if [[ $backup == *.sql.* ]]; then + cmd="$COMMAND_SUDO mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + RemoteLogger "cmd: $cmd" "DEBUG" + eval "$cmd" + if [ $? != 0 ]; then + RemoteLogger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + + elif [ "$REMOTE_OPERATION" == "yes" ]; then + cmd="$COMMAND_SUDO cp -R \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + RemoteLogger "cmd: $cmd" "DEBUG" + eval "$cmd" + if [ $? != 0 ]; then + RemoteLogger "Cannot copy [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + + else + cmd="$COMMAND_SUDO mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + RemoteLogger "cmd: $cmd" "DEBUG" + eval "$cmd" + if [ $? != 0 ]; then + RemoteLogger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + fi + done + IFS=$OLD_IFS +} + + _RotateBackupsRemoteSSH + +ENDSSH + + WaitForTaskCompletion $! 1800 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Could not rotate backups in [$backup_path]." "ERROR" + Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + else + Logger "Remote rotation succeed." "NOTICE" + fi ## Need to add a trivial sleep time to give ssh time to log to local file + #sleep 5 + + +} + +function RotateBackups { + local backup_path="${1}" + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + Logger "Rotating backups." "NOTICE" + + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "pull" ]; then + _RotateBackupsLocal "$backup_path" + elif [ "$BACKUP_TYPE" == "push" ]; then + _RotateBackupsRemote "$backup_path" + fi +} + +function Init { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + trap TrapStop SIGINT SIGQUIT SIGKILL SIGTERM SIGHUP + trap TrapQuit EXIT + + ## Test if target dir is a ssh uri, and if yes, break it down it its values + if [ "${REMOTE_SYSTEM_URI:0:6}" == "ssh://" ] && [ "$BACKUP_TYPE" != "local" ]; then + REMOTE_OPERATION="yes" + + # remove leadng 'ssh://' + uri=${REMOTE_SYSTEM_URI#ssh://*} + if [[ "$uri" == *"@"* ]]; then + # remove everything after '@' + REMOTE_USER=${uri%@*} + else + REMOTE_USER=$LOCAL_USER + fi + + if [ "$SSH_RSA_PRIVATE_KEY" == "" ]; then + SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa + fi + + # remove everything before '@' + _hosturiandpath=${uri#*@} + # remove everything after first '/' + _hosturi=${_hosturiandpath%%/*} + if [[ "$_hosturi" == *":"* ]]; then + REMOTE_PORT=${_hosturi##*:} + else + REMOTE_PORT=22 + fi + REMOTE_HOST=${_hosturi%%:*} + fi + + ## Add update to default RSYNC_ARGS + RSYNC_ARGS=$RSYNC_ARGS"u" + + if [ $_VERBOSE -eq 1 ]; then + RSYNC_ARGS=$RSYNC_ARGS"i" + fi + + if [ "$PARTIAL" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" + RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$PARTIAL_DIR\"" + fi + + if [ "$DELETE_VANISHED_FILES" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --delete" + fi + + if [ "$DELTA_COPIES" != "no" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file" + else + RSYNC_ARGS=$RSYNC_ARGS" --whole-file" + fi + + if [ $stats -eq 1 ]; then + RSYNC_ARGS=$RSYNC_ARGS" --stats" + fi + + ## Fix for symlink to directories on target cannot get updated + RSYNC_ARGS=$RSYNC_ARGS" --force" +} + +function Main { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$SQL_BACKUP" != "no" ] && [ $CAN_BACKUP_SQL -eq 1 ]; then + ListDatabases + fi + if [ "$FILE_BACKUP" != "no" ] && [ $CAN_BACKUP_FILES -eq 1 ]; then + ListRecursiveBackupDirectories + if [ "$GET_BACKUP_SIZE" != "no" ]; then + GetDirectoriesSize + else + TOTAL_FILE_SIZE=0 + fi + fi + + if [ "$CREATE_DIRS" != "no" ]; then + CreateStorageDirectories + fi + CheckDiskSpace + + # Actual backup process + if [ "$SQL_BACKUP" != "no" ] && [ $CAN_BACKUP_SQL -eq 1 ]; then + if [ $_DRYRUN -ne 1 ] && [ "$ROTATE_SQL_BACKUPS" == "yes" ]; then + RotateBackups "$SQL_STORAGE" + fi + BackupDatabases + fi + + if [ "$FILE_BACKUP" != "no" ] && [ $CAN_BACKUP_FILES -eq 1 ]; then + if [ $_DRYRUN -ne 1 ] && [ "$ROTATE_FILE_BACKUPS" == "yes" ]; then + RotateBackups "$FILE_STORAGE" + fi + ## Add Rsync exclude patterns + RsyncExcludePattern + ## Add Rsync exclude from file + RsyncExcludeFrom + FilesBackup + fi +} + +function Usage { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + + if [ "$IS_STABLE" != "yes" ]; then + echo -e "\e[93mThis is an unstable dev build. Please use with caution.\e[0m" + fi + + echo "$PROGRAM $PROGRAM_VERSION $PROGRAM_BUILD" + echo "$AUTHOR" + echo "$CONTACT" + echo "" + echo "usage: obackup.sh /path/to/backup.conf [OPTIONS]" + echo "" + echo "OPTIONS:" + echo "--dry: will run obackup without actually doing anything, just testing" + echo "--silent: will run obackup without any output to stdout, usefull for cron backups" + echo "--verbose: adds command outputs" + echo "--stats Adds rsync transfer statistics to verbose output" + echo "--partial Allows rsync to keep partial downloads that can be resumed later (experimental)" + echo "--no-maxtime disables any soft and hard execution time checks" + echo "--delete Deletes files on destination that vanished on source" + echo "--dontgetsize Does not try to evaluate backup size" + exit 128 +} + +# Command line argument flags +_DRYRUN=0 +_SILENT=0 +no_maxtime=0 +dontgetsize=0 +stats=0 +PARTIAL=0 + +function GetCommandlineArguments { + if [ $# -eq 0 ]; then + Usage + fi + + for i in "$@" + do + case $i in + --dry) + _DRYRUN=1 + ;; + --silent) + _SILENT=1 + ;; + --verbose) + _VERBOSE=1 + ;; + --stats) + stats=1 + ;; + --partial) + PARTIAL="yes" + ;; + --no-maxtime) + no_maxtime=1 + ;; + --delete) + DELETE_VANISHED_FILES="yes" + ;; + --dontgetsize) + GET_BACKUP_SIZE="no" + ;; + --help|-h|--version|-v) + Usage + ;; + esac + done +} + +GetCommandlineArguments "$@" +CheckEnvironment +LoadConfigFile "$1" +if [ "$LOGFILE" == "" ]; then + if [ -w /var/log ]; then + LOG_FILE=/var/log/$PROGRAM.$INSTANCE_ID.log + else + LOG_FILE=./$PROGRAM.$INSTANCE_ID.log + fi +else + LOG_FILE="$LOGFILE" +fi + +if [ "$IS_STABLE" != "yes" ]; then + Logger "This is an unstable dev build. Please use with caution." "WARN" +fi + + +GetLocalOS +InitLocalOSSettings +PreInit +Init +PostInit +CheckCurrentConfig +if [ "$REMOTE_OPERATION" == "yes" ]; then + GetRemoteOS + InitRemoteOSSettings +fi +DATE=$(date) +Logger "--------------------------------------------------------------------" "NOTICE" +Logger "$DRY_WARNING $DATE - $PROGRAM v$PROGRAM_VERSION $BACKUP_TYPE script begin." "NOTICE" +Logger "--------------------------------------------------------------------" "NOTICE" +Logger "Backup instance [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE" + +if [ $no_maxtime -eq 1 ]; then + SOFT_MAX_EXEC_TIME_DB_TASK=0 + SOFT_MAX_EXEC_TIME_FILE_TASK=0 + HARD_MAX_EXEC_TIME_DB_TASK=0 + HARD_MAX_EXEC_TIME_FILE_TASK=0 + HARD_MAX_EXEC_TIME_TOTAL=0 +fi + +RunBeforeHook +Main +RunAfterHook diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh new file mode 100644 index 0000000..b8d624b --- /dev/null +++ b/dev/ofunctions.sh @@ -0,0 +1,885 @@ +FUNC_BUILD=2015111102 +## BEGIN Generic functions for osync & obackup written in 2013-2015 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr + +## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode +if ! type "$BASH" > /dev/null; then + echo "Please run this script only with bash shell. Tested on bash >= 3.2" + exit 127 +fi + +# Environment variables +_DRYRUN=0 +_SILENT=0 + +# Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags +ERROR_ALERT=0 +WARN_ALERT=0 + +## allow function call checks #__WITH_PARANOIA_DEBUG +if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG + _DEBUG=yes #__WITH_PARANOIA_DEBUG +fi #__WITH_PARANOIA_DEBUG + +## allow debugging from command line with _DEBUG=yes +if [ ! "$_DEBUG" == "yes" ]; then + _DEBUG=no + SLEEP_TIME=.1 + _VERBOSE=0 +else + SLEEP_TIME=1 + trap 'TrapError ${LINENO} $?' ERR + _VERBOSE=1 +fi + +SCRIPT_PID=$$ + +LOCAL_USER=$(whoami) +LOCAL_HOST=$(hostname) + +## Default log file until config file is loaded +if [ -w /var/log ]; then + LOG_FILE="/var/log/$PROGRAM.log" +else + LOG_FILE="./$PROGRAM.log" +fi + +## Default directory where to store temporary run files +if [ -w /tmp ]; then + RUN_DIR=/tmp +elif [ -w /var/tmp ]; then + RUN_DIR=/var/tmp +else + RUN_DIR=. +fi + +## Log a state message every $KEEP_LOGGING seconds. Should not be equal to soft or hard execution time so your log will not be unnecessary big. +KEEP_LOGGING=1801 + +## Correct output of sort command (language agnostic sorting) +export LC_ALL=C + +# Standard alert mail body +MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." + +# Default alert attachment filename +ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log" + +# Set error exit code if a piped command fails + set -o pipefail + set -o errtrace + + +function Dummy { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + sleep .1 +} + +function _Logger { + local svalue="${1}" # What to log to screen + local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value + echo -e "$lvalue" >> "$LOG_FILE" + + if [ $_SILENT -eq 0 ]; then + echo -e "$svalue" + fi +} + +function Logger { + local value="${1}" # Sentence to log (in double quotes) + local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL + + # Special case in daemon mode we should timestamp instead of counting seconds + if [ "$sync_on_changes" == "1" ]; then + prefix="$(date) - " + else + prefix="TIME: $SECONDS - " + fi + # + + if [ "$level" == "CRITICAL" ]; then + _Logger "$prefix\e[41m$value\e[0m" "$prefix$value" + ERROR_ALERT=1 + return + elif [ "$level" == "ERROR" ]; then + _Logger "$prefix\e[91m$value\e[0m" "$prefix$value" + ERROR_ALERT=1 + return + elif [ "$level" == "WARN" ]; then + _Logger "$prefix\e[93m$value\e[0m" "$prefix$value" + WARN_ALERT=1 + return + elif [ "$level" == "NOTICE" ]; then + _Logger "$prefix$value" + return + elif [ "$level" == "DEBUG" ]; then + if [ "$_DEBUG" == "yes" ]; then + _Logger "$prefix$value" + return + fi + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG + _Logger "$prefix$value" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG + else + _Logger "\e[41mLogger function called without proper loglevel.\e[0m" + _Logger "$prefix$value" + fi +} + +# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X +function KillChilds { + local pid="${1}" + local self="${2:-false}" + + if children="$(pgrep -P "$pid")"; then + for child in $children; do + KillChilds "$child" true + done + fi + + # Try to kill nicely, if not, wait 30 seconds to let Trap actions happen before killing + if [ "$self" == true ]; then + kill -s SIGTERM "$pid" || (sleep 30 && kill -9 "$pid" &) + fi +} + +function TrapError { + local job="$0" + local line="$1" + local code="${2:-1}" + if [ $_SILENT -eq 0 ]; then + echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}" + fi +} + +function Spinner { + if [ $_SILENT -eq 1 ]; then + return 0 + fi + + case $toggle + in + 1) + echo -n " \ " + echo -ne "\r" + toggle="2" + ;; + + 2) + echo -n " | " + echo -ne "\r" + toggle="3" + ;; + + 3) + echo -n " / " + echo -ne "\r" + toggle="4" + ;; + + *) + echo -n " - " + echo -ne "\r" + toggle="1" + ;; + esac +} + +function SedStripQuotes { + echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") +} + +function StripSingleQuotes { + local string="${1}" + string="${string/#\'/}" # Remove singlequote if it begins string + string="${string/%\'/}" # Remove singlequote if it ends string + echo "$string" +} + +function StripDoubleQuotes { + local string="${1}" + string="${string/#\"/}" + string="${string/%\"/}" + echo "$string" +} + +function StripQuotes { + local string="${1}" + echo "$(StripSingleQuotes $(StripDoubleQuotes $string))" +} + +function EscapeSpaces { + local string="${1}" # String on which spaces will be escaped + echo "${string// /\ }" +} + +function IsNumeric { + eval "local value=\"${1}\"" # Needed so variable variables can be processed + + local re="^-?[0-9]+([.][0-9]+)?$" + if [[ $value =~ $re ]]; then + echo 1 + else + echo 0 + fi +} + +function CleanUp { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$_DEBUG" != "yes" ]; then + rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID" + fi +} + +function SendAlert { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$_DEBUG" == "yes" ]; then + Logger "Debug mode, no warning email will be sent." "NOTICE" + return 0 + fi + + # + if [ "$_QUICK_SYNC" == "2" ]; then + Logger "Current task is a quicksync task. Will not send any alert." "NOTICE" + return 0 + fi + # + + eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE" + MAIL_ALERT_MSG="$MAIL_ALERT_MSG"$'\n\n'$(tail -n 25 "$LOG_FILE") + if [ $ERROR_ALERT -eq 1 ]; then + subject="Error alert for $INSTANCE_ID" + elif [ $WARN_ALERT -eq 1 ]; then + subject="Warning alert for $INSTANCE_ID" + else + subject="Alert for $INSTANCE_ID" + fi + + # Need better fallback if mail sending does not succeed + if type mutt > /dev/null 2>&1 ; then + echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS -a "$ALERT_LOG_FILE" + if [ $? != 0 ]; then + Logger "WARNING: Cannot send alert email via $(type -p mutt) !!!" "WARN" + else + Logger "Sent alert mail using mutt." "NOTICE" + return 0 + fi + fi + + if type mail > /dev/null 2>&1 ; then + echo "$MAIL_ALERT_MSG" | $(type -p mail) -a "$ALERT_LOG_FILE" -s "$subject" $DESTINATION_MAILS + if [ $? != 0 ]; then + Logger "WARNING: Cannot send alert email via $(type -p mail) with attachments !!!" "WARN" + echo "$MAIL_ALERT_MSG" | $(type -p mail) -s "$subject" $DESTINATION_MAILS + if [ $? != 0 ]; then + Logger "WARNING: Cannot send alert email via $(type -p mail) without attachments !!!" "WARN" + else + Logger "Sent alert mail using mail command without attachment." "NOTICE" + return 0 + fi + else + Logger "Sent alert mail using mail command." "NOTICE" + return 0 + fi + fi + + if type sendmail > /dev/null 2>&1 ; then + echo -e "$subject\r\n$MAIL_ALERT_MSG" | $(type -p sendmail) $DESTINATION_MAILS + if [ $? != 0 ]; then + Logger "WARNING: Cannot send alert email via $(type -p sendmail) !!!" "WARN" + else + Logger "Sent alert mail using sendmail command without attachment." "NOTICE" + return 0 + fi + fi + + if type sendemail > /dev/null 2>&1 ; then + if [ "$SMTP_USER" != "" ] && [ "$SMTP_PASSWORD" != "" ]; then + SMTP_OPTIONS="-xu $SMTP_USER -xp $SMTP_PASSWORD" + else + SMTP_OPTIONS="" + fi + $(type -p sendemail) -f $SENDER_MAIL -t $DESTINATION_MAILS -u "$subject" -m "$MAIL_ALERT_MSG" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1 + if [ $? != 0 ]; then + Logger "WARNING: Cannot send alert email via $(type -p sendemail) !!!" "WARN" + else + Logger "Sent alert mail using sendemail command without attachment." "NOTICE" + return 0 + fi + fi + + # If function has not returned 0 yet, assume it's critical that no alert can be sent + Logger "/!\ CRITICAL: Cannot send alert" "ERROR" # Is not marked critical because execution must continue + + # Delete tmp log file + if [ -f "$ALERT_LOG_FILE" ]; then + rm "$ALERT_LOG_FILE" + fi +} + +function LoadConfigFile { + __CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + local config_file="${1}" + + if [ ! -f "$config_file" ]; then + Logger "Cannot load configuration file [$config_file]. Cannot start." "CRITICAL" + exit 1 + elif [[ "$1" != *".conf" ]]; then + Logger "Wrong configuration file supplied [$config_file]. Cannot start." "CRITICAL" + exit 1 + else + grep '^[^ ]*=[^;&]*' "$config_file" > "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" # WITHOUT COMMENTS + # Shellcheck source=./sync.conf + source "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" + fi +} + +function GetLocalOS { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + local local_os_var=$(uname -spio 2>&1) + if [ $? != 0 ]; then + local local_os_var=$(uname -v 2>&1) + if [ $? != 0 ]; then + local local_os_var=($uname) + fi + fi + + case $local_os_var in + *"Linux"*) + LOCAL_OS="Linux" + ;; + *"BSD"*) + LOCAL_OS="BSD" + ;; + *"MINGW32"*) + LOCAL_OS="msys" + ;; + *"Darwin"*) + LOCAL_OS="MacOSX" + ;; + *) + Logger "Running on >> $local_os_var << not supported. Please report to the author." "ERROR" + exit 1 + ;; + esac + Logger "Local OS: [$local_os_var]." "DEBUG" +} + +function GetRemoteOS { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$REMOTE_OPERATION" == "yes" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + local cmd=$SSH_CMD' "uname -spio" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 120 240 $FUNCNAME"-1" + retval=$? + if [ $retval != 0 ]; then + local cmd=$SSH_CMD' "uname -v" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 120 240 $FUNCNAME"-2" + retval=$? + if [ $retval != 0 ]; then + local cmd=$SSH_CMD' "uname" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 120 240 $FUNCNAME"-3" + retval=$? + if [ $retval != 0 ]; then + Logger "Cannot Get remote OS type." "ERROR" + fi + fi + fi + + local remote_os_var=$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID) + + case $remote_os_var in + *"Linux"*) + REMOTE_OS="Linux" + ;; + *"BSD"*) + REMOTE_OS="BSD" + ;; + *"MINGW32"*) + REMOTE_OS="msys" + ;; + *"Darwin"*) + REMOTE_OS="MacOSX" + ;; + *"ssh"*|*"SSH"*) + Logger "Cannot connect to remote system." "CRITICAL" + exit 1 + ;; + *) + Logger "Running on remote OS failed. Please report to the author if the OS is not supported." "CRITICAL" + Logger "Remote OS said:\n$remote_os_var" "CRITICAL" + exit 1 + esac + + Logger "Remote OS: [$remote_os_var]." "DEBUG" + fi +} + +function WaitForTaskCompletion { + local pid="${1}" # pid to wait for + local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. + local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. + local caller_name="${4}" # Who called this function + Logger "$FUNCNAME called by [$caller_name]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + __CheckArguments 4 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once + local log_ttime=0 # local time instance for comparaison + + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function + + while eval "$PROCESS_TEST_CMD" > /dev/null + do + Spinner + exec_time=$(($SECONDS - $seconds_begin)) + if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then + if [ $log_ttime -ne $exec_time ]; then + log_ttime=$exec_time + Logger "Current task still running." "NOTICE" + fi + fi + if [ $exec_time -gt $soft_max_time ]; then + if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then + Logger "Max soft execution time exceeded for task." "WARN" + soft_alert=1 + SendAlert + + fi + if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then + Logger "Max hard execution time exceeded for task. Stopping task execution." "ERROR" + kill -s SIGTERM $pid + if [ $? == 0 ]; then + Logger "Task stopped succesfully" "NOTICE" + else + Logger "Sending SIGTERM to proces failed. Trying the hard way." "ERROR" + sleep 5 && kill -9 $pid + if [ $? != 0 ]; then + Logger "Could not stop task." "ERROR" + fi + fi + return 1 + fi + fi + sleep $SLEEP_TIME + done + wait $pid + local retval=$? + Logger "$FUNCNAME ended for [$caller_name] with status $retval." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + return $retval +} + +function WaitForCompletion { + local pid="${1}" # pid to wait for + local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. + local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. + local caller_name="${4}" # Who called this function + Logger "$FUNCNAME called by [$caller_name]" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + __CheckArguments 4 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once + local log_ttime=0 # local time instance for comparaison + + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function + + while eval "$PROCESS_TEST_CMD" > /dev/null + do + Spinner + if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ]; then + if [ $log_time -ne $SECONDS ]; then + log_time=$SECONDS + Logger "Current task still running." "NOTICE" + fi + fi + if [ $SECONDS -gt $soft_max_time ]; then + if [ $soft_alert -eq 0 ] && [ $soft_max_time != 0 ]; then + Logger "Max soft execution time exceeded for script." "WARN" + soft_alert=1 + SendAlert + fi + if [ $SECONDS -gt $hard_max_time ] && [ $hard_max_time != 0 ]; then + Logger "Max hard execution time exceeded for script. Stopping current task execution." "ERROR" + kill -s SIGTERM $pid + if [ $? == 0 ]; then + Logger "Task stopped succesfully" "NOTICE" + else + Logger "Sending SIGTERM to proces failed. Trying the hard way." "ERROR" + kill -9 $pid + if [ $? != 0 ]; then + Logger "Could not stop task." "ERROR" + fi + fi + return 1 + fi + fi + sleep $SLEEP_TIME + done + wait $pid + retval=$? + Logger "$FUNCNAME ended for [$caller_name] with status $retval." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + return $retval +} + +function RunLocalCommand { + local command="${1}" # Command to run + local hard_max_time="${2}" # Max time to wait for command to compleet + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ $_DRYRUN -ne 0 ]; then + Logger "Dryrun: Local command [$command] not run." "NOTICE" + return 1 + fi + Logger "Running command [$command] on local host." "NOTICE" + eval "$command" > "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" 2>&1 & + WaitForTaskCompletion $! 0 $hard_max_time $FUNCNAME + retval=$? + if [ $retval -eq 0 ]; then + Logger "Command succeded." "NOTICE" + else + Logger "Command failed." "ERROR" + fi + + if [ $_VERBOSE -eq 1 ] || [ $retval -ne 0 ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE" + fi + + if [ "$STOP_ON_CMD_ERROR" == "yes" ] && [ $retval -ne 0 ]; then + Logger "Stopping on command execution error." "CRITICAL" + exit 1 + fi +} + +## Runs remote command $1 and waits for completition in $2 seconds +function RunRemoteCommand { + local command="${1}" # Command to run + local hard_max_time="${2}" # Max time to wait for command to compleet + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + if [ $_DRYRUN -ne 0 ]; then + Logger "Dryrun: Local command [$command] not run." "NOTICE" + return 1 + fi + Logger "Running command [$command] on remote host." "NOTICE" + cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 0 $hard_max_time $FUNCNAME + retval=$? + if [ $retval -eq 0 ]; then + Logger "Command succeded." "NOTICE" + else + Logger "Command failed." "ERROR" + fi + + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ] && ([ $_VERBOSE -eq 1 ] || [ $retval -ne 0 ]) + then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE" + fi + + if [ "$STOP_ON_CMD_ERROR" == "yes" ] && [ $retval -ne 0 ]; then + Logger "Stopping on command execution error." "CRITICAL" + exit 1 + fi +} + +function RunBeforeHook { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then + RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE + fi + + if [ "$REMOTE_RUN_BEFORE_CMD" != "" ]; then + RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE + fi +} + +function RunAfterHook { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$LOCAL_RUN_AFTER_CMD" != "" ]; then + RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER + fi + + if [ "$REMOTE_RUN_AFTER_CMD" != "" ]; then + RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER + fi +} + +function CheckConnectivityRemoteHost { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug + + if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then + eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & + WaitForTaskCompletion $! 180 180 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot ping $REMOTE_HOST" "CRITICAL" + return 1 + fi + fi + fi +} + +function CheckConnectivity3rdPartyHosts { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug + + if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then + remote_3rd_party_success=0 + OLD_IFS=$IFS + IFS=$' \t\n' + for i in $REMOTE_3RD_PARTY_HOSTS + do + eval "$PING_CMD $i > /dev/null 2>&1" & + WaitForTaskCompletion $! 360 360 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot ping 3rd party host $i" "WARN" + else + remote_3rd_party_success=1 + fi + done + IFS=$OLD_IFS + if [ $remote_3rd_party_success -ne 1 ]; then + Logger "No remote 3rd party host responded to ping. No internet ?" "CRITICAL" + return 1 + fi + fi + fi +} + +#__BEGIN_WITH_PARANOIA_DEBUG +function __CheckArguments { + # Checks the number of arguments of a function and raises an error if some are missing + + if [ "$_DEBUG" == "yes" ]; then + local number_of_arguments="${1}" # Number of arguments the tested function should have + local number_of_given_arguments="${2}" # Number of arguments that have been passed + local function_name="${3}" # Function name that called __CheckArguments + + if [ "$_PARANOIA_DEBUG" == "yes" ]; then + Logger "Entering function [$function_name]." "DEBUG" + fi + + # All arguments of the function to check are passed as array in ${4} (the function call waits for $@) + # If any of the arguments contains spaces, bash things there are two aguments + # In order to avoid this, we need to iterate over ${4} and count + + local iterate=4 + local fetch_arguments=1 + local arg_list="" + while [ $fetch_arguments -eq 1 ]; do + cmd='argument=${'$iterate'}' + eval $cmd + if [ "$argument" = "" ]; then + fetch_arguments=0 + else + arg_list="$arg_list [Argument $(($iterate-3)): $argument]" + iterate=$(($iterate+1)) + fi + done + local counted_arguments=$((iterate-4)) + + if [ $counted_arguments -ne $number_of_arguments ]; then + Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $counted_arguments, see log file." "ERROR" + Logger "Arguments passed: $arg_list" "ERROR" + fi + fi +} + + +function old__CheckArguments { + # Checks the number of arguments and raises an error if some are missing + if [ "$_DEBUG" == "yes" ]; then + + local number_of_arguments="${1}" # Number of arguments a function should have + local number_of_given_arguments="${2}" # Number of arguments that have been passed + local function_name="${3}" # Function name that called __CheckArguments + local arguments="${4}" # All other arguments + + if [ "$_PARANOIA_DEBUG" == "yes" ]; then + Logger "Entering function [$function_name]." "DEBUG" + + # Paranoia check... Can help finding empty arguments. __CheckArguments should be grepped out in production builds. + local count=-3 # Number of arguments minus the function calls for __CheckArguments + for i in $@; do + count=$((count + 1)) + done + if [ $count -ne $1 ]; then + Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $count, see log file." "WARN" + echo "Argument list (including checks): $*" >> "$LOG_FILE" + fi + fi + + if [ $number_of_arguments -ne $number_of_given_arguments ]; then + Logger "Inconsistnent number of arguments in $function_name. Should have $number_of_arguments arguments, has $number_of_given_arguments arguments, see log file." "CRITICAL" + # Cannot user Logger here because $@ is a list of arguments + echo "Argumnt list: $4" >> "$LOG_FILE" + fi + + fi +} +#__END_WITH_PARANOIA_DEBUG + +function PreInit { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + ## SSH compression + if [ "$SSH_COMPRESSION" != "no" ]; then + SSH_COMP=-C + else + SSH_COMP= + fi + + ## Support for older config files without RSYNC_EXECUTABLE option + if [ "$RSYNC_EXECUTABLE" == "" ]; then + RSYNC_EXECUTABLE=rsync + fi + + ## Sudo execution option + if [ "$SUDO_EXEC" == "yes" ]; then + if [ "$RSYNC_REMOTE_PATH" != "" ]; then + RSYNC_PATH="sudo $RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" + else + RSYNC_PATH="sudo $RSYNC_EXECUTABLE" + fi + COMMAND_SUDO="sudo" + else + if [ "$RSYNC_REMOTE_PATH" != "" ]; then + RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" + else + RSYNC_PATH="$RSYNC_EXECUTABLE" + fi + COMMAND_SUDO="" + fi + + ## Set rsync default arguments + RSYNC_ARGS="-rlptgoD" + + if [ "$PRESERVE_ACL" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -A" + fi + if [ "$PRESERVE_XATTR" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -X" + fi + if [ "$RSYNC_COMPRESS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -z" + fi + if [ "$COPY_SYMLINKS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -L" + fi + if [ "$KEEP_DIRLINKS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -K" + fi + if [ "$PRESERVE_HARDLINKS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -H" + fi + if [ "$CHECKSUM" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --checksum" + fi + if [ $_DRYRUN -eq 1 ]; then + RSYNC_ARGS=$RSYNC_ARGS" -n" + DRY_WARNING="/!\ DRY RUN" + fi + if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" + fi + + ## Set compression executable and extension + COMPRESSION_LEVEL=3 + if type xz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| xz -$COMPRESSION_LEVEL" + COMPRESSION_EXTENSION=.xz + elif type lzma > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| lzma -$COMPRESSION_LEVEL" + COMPRESSION_EXTENSION=.lzma + elif type pigz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| pigz -$COMPRESSION_LEVEL" + COMPRESSION_EXTENSION=.gz + COMPRESSION_OPTIONS=--rsyncable + elif type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -$COMPRESSION_LEVEL" + COMPRESSION_EXTENSION=.gz + COMPRESSION_OPTIONS=--rsyncable + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" +} + +function PostInit { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + # Define remote commands + SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" + SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" + RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -p $REMOTE_PORT" +} + +function InitLocalOSSettings { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + ## If running under Msys, some commands do not run the same way + ## Using mingw version of find instead of windows one + ## Getting running processes is quite different + ## Ping command is not the same + if [ "$LOCAL_OS" == "msys" ]; then + FIND_CMD=$(dirname $BASH)/find + #TODO: The following command needs to be checked on msys. Does the $1 variable substitution work ? + # PROCESS_TEST_CMD assumes there is a variable $pid + PROCESS_TEST_CMD='ps -a | awk "{\$1=\$1}\$1" | awk "{print \$1}" | grep $pid' + PING_CMD="ping -n 2" + else + FIND_CMD=find + # PROCESS_TEST_CMD assumes there is a variable $pid + PROCESS_TEST_CMD='ps -p$pid' + PING_CMD="ping -c 2 -i .2" + fi + + ## Stat command has different syntax on Linux and FreeBSD/MacOSX + if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then + STAT_CMD="stat -f \"%Sm\"" + else + STAT_CMD="stat --format %y" + fi +} + +function InitRemoteOSSettings { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -E" + fi + + if [ "$REMOTE_OS" == "msys" ]; then + REMOTE_FIND_CMD=$(dirname $BASH)/find + else + REMOTE_FIND_CMD=find + fi +} + +## END Generic functions diff --git a/exclude.list.example b/exclude.list.example deleted file mode 100644 index 08813fc..0000000 --- a/exclude.list.example +++ /dev/null @@ -1,7 +0,0 @@ -.AppleDouble/ -._* -.DS_Store -Thumbs.db -System Volume Information -$Recycle.Bin - diff --git a/exclude.list.example b/exclude.list.example new file mode 120000 index 0000000..8cef884 --- /dev/null +++ b/exclude.list.example @@ -0,0 +1 @@ +/home/git/common/exclude.list.example \ No newline at end of file diff --git a/host_backup.conf b/host_backup.conf index 9b5f623..917622c 100755 --- a/host_backup.conf +++ b/host_backup.conf @@ -1,83 +1,86 @@ -#!/bin/bash +###### Local / Remote backup script for files & databases +###### (L) 2013-2015 by Ozy de Jong (www.netpower.fr) +###### obackup v2.x config file rev 2015111102 -###### Remote (or local) backup script for files & databases -###### (L) 2013 by Ozy de Jong (www.netpower.fr) -###### Config file rev 2015090801 +###### GENERAL BACKUP OPTIONS -## ---------- GENERAL BACKUP OPTIONS +## Backup identification string. +INSTANCE_ID="test-backup" -## Backup identification name. -BACKUP_ID="your backup name" - -## Log file location. Leaving this empty will create log file at /var/log/obackup_version_BACKUP_ID.log (or current directory if /var/log doesn't exist) +## Log file location. Leaving this empty will create log file at /var/log/obackup.INSTANCE_ID.log (or current directory if /var/log doesn't exist). LOGFILE="" -## Backup databases -BACKUP_SQL=no -## Backup files -BACKUP_FILES=yes +## Elements to backup +SQL_BACKUP=yes +FILE_BACKUP=yes -## ---------- LOCAL BACKUP STORAGE OPTIONS +## Backups can be done local, pulled from another server or pushed to a backup server. Available options are [local,pull,push]. +## Pulled backups are the safest option, as the backup server contains the RSA key and cannot be compromised by another server. +BACKUP_TYPE=local -## Local storage paths where to put backups -LOCAL_SQL_STORAGE="/home/storage/backup/sql" -LOCAL_FILE_STORAGE="/home/storage/backup/files" +###### BACKUP STORAGE + +## Storage paths of the backups (absolute paths of the local or remote system) +SQL_STORAGE="/home/storage/backup/sql" +FILE_STORAGE="/home/storage/backup/files" + +## Backup encryption using GPG and duplicity. Feature not ready yet. +ENCRYPTION=no ## Create backup directories if they do not exist CREATE_DIRS=yes ## Keep absolute source path in your backup, eg: /your/backup/storage/the/remote/server/files -## You should leave this enabled if you intend to use 'backup task division' functionality of OBackup, or everything will end up in the same directory. -LOCAL_STORAGE_KEEP_ABSOLUTE_PATHS=yes +## You should leave this enabled if you intend to use 'backup task division' functionality of oBackup, or everything will end up in the same directory. +KEEP_ABSOLUTE_PATHS=yes -## Generate an alert if backup size is lower than given value in Kb (this can also help identifying empty mount dirs) +## Generate an alert if backup size is lower than given value in Kb (this can also help identifying empty mount dirs). BACKUP_SIZE_MINIMUM=1024 -## You may disable testing backup size -DISABLE_GET_BACKUP_FILE_SIZE=no -## Generate an alert if local storage free space is lower than given value in Kb. Keep in mind that disabling backup file size test will only test min space against SQL backup size. -LOCAL_STORAGE_WARN_MIN_SPACE=1048576 +## Check backup size before proceeding +GET_BACKUP_SIZE=yes -## ---------- MISC OPTIONS +## Generate an alert if storage free space is lower than given value in Kb. +## Keep in mind that disabling backup file size test will only test min space against SQL backup size. +SQL_WARN_MIN_SPACE=1048576 +FILE_WARN_MIN_SPACE=1048576 -## Bandwidth limit Kbytes / second for file backups. Leave 0 to disable limitation. -BANDWIDTH=0 +## If enabled, backups will be processed as superuser. See documentation for /etc/sudoers configuration ("find", "du", "tee" and "rsync" need to be allowed). Requiretty needs to be disabled. +SUDO_EXEC=yes -## If enabled, file backups will be processed as superuser. See documentation for /etc/sudoers configuration ("find", "du" and "rsync" need to be allowed). Requiretty needs to be disabled. -SUDO_EXEC=no -## Paranoia option. Don't change this unless you read the documentation. -RSYNC_EXECUTABLE=rsync +###### REMOTE ONLY OPTIONS -## ---------- REMOTE BACKUP OPTIONS +## In case of pulled or pushed backups, remote system URI needs to be supplied. +#REMOTE_SYSTEM_URI="ssh://backupuser@remote.system.tld:22/" +REMOTE_SYSTEM_URI="ssh://backupmaster@mailer.mvacances.org:49998/" + +## You can specify a RSA key (please use full path). If not defined, the default ~/.ssh/id_rsa will be used. See documentation for further information. +SSH_RSA_PRIVATE_KEY="/root/.ssh/id_rsa.srvmailing" -## The following options allow this Obackup instance to connect to a remote system via an ssh tunnel. -## Needs public RSA key need to be put into ~/.ssh/authorized_keys in remote users home directory. -REMOTE_BACKUP=no -SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa -REMOTE_USER=backupuser -REMOTE_HOST=yourhost.local -REMOTE_PORT=22 ## ssh compression should be used unless your remote connection is good enough (LAN) SSH_COMPRESSION=yes + ## Remote rsync executable path. Leave this empty in most cases RSYNC_REMOTE_PATH="" ## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task. REMOTE_HOST_PING=yes + ## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task. REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com" -## ---------- DATABASE BACKUP OPTIONS +###### DATABASE SPECIFIC OPTIONS ## Database backup user -SQL_USER=backupuser +SQL_USER=backupmaster +SQL_USER=root #DEBUG ## Enabling the following option will save all databases on local or remote given SQL instance except the ones specified in the exclude list. ## Every found database will be backed up as separate backup task. DATABASES_ALL=yes -DATABASES_ALL_EXCLUDE_LIST="test;mysql" +DATABASES_ALL_EXCLUDE_LIST="test;phplistdb_prod" #DEBUG ## Alternatively, if DATABASES_ALL=no, you can specify a list of databases to backup separated by spaces. -DATABASES_LIST="" +#DATABASES_LIST="somedatabase" ## Max backup execution time per Database task. Soft max exec time generates a warning only. Hard max exec time generates a warning and stops current backup task. ## If a task gets stopped, next one in the task list gets executed. Time is specified in seconds. @@ -88,31 +91,24 @@ HARD_MAX_EXEC_TIME_DB_TASK=7200 ## Generally, level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable. COMPRESSION_LEVEL=3 -## SQL Dump compression should be done on remote side but can also be done locally to lower remote system usage (will take more bandwidth, check for ssh compression) -COMPRESSION_REMOTE=yes +###### FILES SPECIFIC OPTIONS -## ---------- FILES BACKUP OPTIONS +## File backups are divided in tasks. Every directory in DIRECTORY_LIST will be processed as a unique task. +## Every subdirectory of each directory in RECURSIVE_DIRECTORY_LIST will be processed as a unique task. +## Example: RECURSIVE_DIRECTORY_LIST="/home;/var" will create backup tasks tasks "/home/dir1, "/home/dir2", ... "/home/dirN", "/var/log", "/var/lib"... "/var/something". +## You can exclude directories from the avove backup task creation, ex: avoid backing up "/home/dir2" by adding it to RECURSIVE_EXCLUDE_LIST. -## Directories backup list. List of semicolon separated directories that will be backed up recursively. Every directory will be processed as one backup task. -DIRECTORIES_SIMPLE_LIST="/var/named;/var/lib" +## Directories backup list. List of semicolon separated directories that will be backed up. +DIRECTORY_LIST="/var/named" +RECURSIVE_DIRECTORY_LIST="/home" +RECURSIVE_EXCLUDE_LIST="/home/backupuser;/home/lost+found" -## There's a special backup schema in Obackup called 'backup task division' which creates one backup task per level 1 subdirectory of a directory. -## This is VERY useful to backup multiple virtualhosts as separate tasks without having to specify each one separately. -## This may also be useful dividing big data directories in subdirectories tasks. - -## Directories backup task division backup: Semicolon separated directories of which every level 1 subdirectory will be backed up recursively as a separate backup task. -## Example: "/home;/var" will create tasks "/home/dir1", "/home/dir2", ... "/home/dirN", "/var/log", "/var/lib"... "/var/whatever" -DIRECTORIES_RECURSE_LIST="/home" -## You may optionally exclude subdirectories from task division. On the above example you could exclude /home/dir2 by adding it to DIRECTORIES_RECURSE_EXCLUDE_LIST -DIRECTORIES_RECURSE_EXCLUDE_LIST="/home/backupuser;/home/lost+found" - -## Rsync exclude patterns, used by simple and division lists +## Rsync exclude patterns, used by simple and division lists, separated by semicolons RSYNC_EXCLUDE_PATTERN="*/tmp;*/ftp/www/cache/cachefs;*/sessions" ## File that contains the list of directories or files to exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file. ## This file has to be in the same directory as the config file ## Paths are relative to sync dirs. One element per line. -RSYNC_EXCLUDE_FROM="" #RSYNC_EXCLUDE_FROM="exclude.list" ## List separator char. You may set an alternative separator char for your directories lists above. @@ -147,7 +143,13 @@ DELETE_VANISHED_FILES=no ## Use delta copy algortithm (usefull when local paths are network drives), defaults to yes DELTA_COPIES=yes -## ---------- ALERT OPTIONS +## Bandwidth limit Kbytes / second for file backups. Leave 0 to disable limitation. +BANDWIDTH=0 + +## Paranoia option. Don't change this unless you read the documentation. +RSYNC_EXECUTABLE=rsync + +###### ALERT OPTIONS ## Alert email addresses separated by a space character DESTINATION_MAILS="your@mail.address" @@ -158,7 +160,7 @@ SMTP_SERVER=smtp.your.isp.com SMTP_USER= SMTP_PASSWORD= -## ---------- GENERAL BACKUP OPTIONS +###### GENERAL BACKUP OPTIONS ## Max execution time of whole backup process. Soft max exec time generates a warning only. ## Hard max exec time generates a warning and stops the whole backup execution. @@ -166,10 +168,11 @@ SOFT_MAX_EXEC_TIME_TOTAL=30000 HARD_MAX_EXEC_TIME_TOTAL=36000 ## Backup Rotation. You may rotate backups if you don't use snapshots on your backup server. -ROTATE_BACKUPS=no +ROTATE_SQL_BACKUPS=no +ROTATE_FILE_BACKUPS=no ROTATE_COPIES=7 -## ---------- EXECUTION HOOKS +###### EXECUTION HOOKS ## Commands can will be run before and / or after backup execution (remote execution will only happen if REMOTE_BACKUP is set). ## This is useful to make a snapshot before backing up data, or even handle snapshots of backed up data. diff --git a/install.sh b/install.sh index a88d7fe..f6e4d68 100755 --- a/install.sh +++ b/install.sh @@ -1,25 +1,77 @@ #!/usr/bin/env bash -SCRIPT_BUILD=2015082501 +PROGRAM=obackup +PROGRAM_BINARY=$PROGRAM".sh" +PROGRAM_BATCH=$PROGRAM"-batch.sh" +SCRIPT_BUILD=2015102701 -## Obackup install script -## Tested on RHEL / CentOS 6 & 7 +## osync / obackup daemon install script +## Tested on RHEL / CentOS 6 & 7 and Mint 17 ## Please adapt this to fit your distro needs -if [ "$(whoami)" != "root" ] -then +CONF_DIR=/etc/$PROGRAM +BIN_DIR=/usr/local/bin +SERVICE_DIR=/etc/init.d + +if [ "$(whoami)" != "root" ]; then echo "Must be run as root." exit 1 fi -mkdir /etc/obackup -cp ./host_backup.conf /etc/obackup/host_backup.conf.example -cp ./exclude.list.example /etc/obackup -cp ./obackup.sh /usr/local/bin -cp ./obackup-batch.sh /usr/local/bin +if [ ! -d "$CONF_DIR" ]; then + mkdir "$CONF_DIR" + if [ $? == 0 ]; then + echo "Created directory [$CONF_DIR]." + else + echo "Cannot create directory [$CONF_DIR]." + exit 1 + fi +else + echo "Config directory [$CONF_DIR] exists." +fi + +if [ -f ./sync.conf ]; then + cp ./sync.conf /etc/$PROGRAM/sync.conf.example +fi + +if [ -f ./host_backup.conf ]; then + cp ./host_backup.conf /etc/$PROGRAM/host_backup.conf.example +fi + +if [ -f ./exlude.list.example ]; then + cp ./exclude.list.example /etc/$PROGRAM +fi + +cp ./$PROGRAM_BINARY "$BIN_DIR" +if [ $? != 0 ]; then + echo "Cannot copy $PROGRAM_BINARY to [$BIN_DIR]." +else + echo "Copied $PROGRAM_BINARY to [$BIN_DIR]." +fi +cp ./$PROGRAM_BATCH /usr/local/bin +if [ $? != 0 ]; then + echo "Cannot copy $PROGRAM_BATCH to [$BIN_DIR]." +else + echo "Copied $PROGRAM_BATCH to [$BIN_DIR]." +fi cp ./ssh_filter.sh /usr/local/bin -chmod 755 /usr/local/bin/obackup.sh -chmod 755 /usr/local/bin/obackup-batch.sh +if [ $? != 0 ]; then + echo "Cannot copy ssh_filter.sh to [$BIN_DIR]." +else + echo "Copied ssh_filter.sh to [$BIN_DIR]." +fi + +if [ -f ./osync-srv ]; then + cp ./osync-srv "$SERVICE_DIR" + if [ $? != 0 ]; then + echo "Cannot copy osync-srv to [$SERVICE_DIR]." + else + echo "Created osync-srv service in [$SERVICE_DIR]." + chmod 755 /etc/init.d/osync-srv + fi +fi + +chmod 755 /usr/local/bin/$PROGRAM_BINARY +chmod 755 /usr/local/bin/$PROGRAM_BATCH chmod 755 /usr/local/bin/ssh_filter.sh chown root:root /usr/local/bin/ssh_filter.sh - diff --git a/obackup-batch.sh b/obackup-batch.sh index 72d1d59..c53765a 100755 --- a/obackup-batch.sh +++ b/obackup-batch.sh @@ -1,15 +1,15 @@ #!/usr/bin/env bash +SUBPROGRAM=obackup +PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones +AUTHOR="(L) 2013-2015 by Orsiris \"Ozy\" de Jong" +CONTACT="http://www.netpower.fr - ozy@netpower.fr" +PROGRAM_BUILD=2015103001 -PROGRAM="Obackup-batch" # Batch program to run obackup instances sequentially and rerun failed ones -AUTHOR="(L) 2013-2014 by Orsiris \"Ozy\" de Jong" -CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" -PROGRAM_BUILD=2508201501 - -## Runs an obackup instance for every conf file found +## Runs an osync /obackup instance for every conf file found ## If an instance fails, run it again if time permits -## Configuration file path. The path where all the obackup conf files are, usually /etc/obackup -CONF_FILE_PATH=/etc/obackup +## Configuration file path. The path where all the osync / obackup conf files are, usually /etc/osync or /etc/obackup +CONF_FILE_PATH=/etc/$SUBPROGRAM ## If maximum execution time is not reached, failed instances will be rerun. Max exec time is in seconds. Example is set to 10 hours. MAX_EXECUTION_TIME=36000 @@ -18,60 +18,76 @@ MAX_EXECUTION_TIME=36000 MAX_RERUNS=3 ## Log file path -if [ -w /var/log ] -then - LOG_FILE=/var/log/obackup-batch.log +if [ -w /var/log ]; then + LOG_FILE=/var/log/$SUBPROGRAM-batch.log else - LOG_FILE=./obackup-batch.log + LOG_FILE=./$SUBPROGRAM-batch.log fi # No need to edit under this line ############################################################## -function Log -{ - prefix="TIME: $SECONDS - " - echo -e "$prefix$1" >> "$LOG_FILE" +function _logger { + local value="${1}" # What to log + echo -e "$value" >> "$LOG_FILE" - if [ $silent -eq 0 ] - then - echo -e "$prefix$1" - fi + if [ $_SILENT -eq 0 ]; then + echo -e "$value" + fi } -function CheckEnvironment -{ - ## Obackup executable full path can be set here if it cannot be found on the system - if ! type -p obackup.sh > /dev/null 2>&1 - then - if [ -f /usr/local/bin/obackup.sh ] - then - OBACKUP_EXECUTABLE=/usr/local/bin/obackup.sh - elif [ -f ./obackup.sh ] - then - OBACKUP_EXECUTABLE=./obackup.sh - else - Log "Could not find obackup.sh" - exit 1 - fi - else - OBACKUP_EXECUTABLE=$(type -p obackup.sh) - fi +function Logger { + local value="${1}" # What to log + local level="${2}" # Log level: DEBUG, NOTICE, WARN, ERROR, CRITIAL + + prefix="$(date) - " + + if [ "$level" == "CRITICAL" ]; then + _logger "$prefix\e[41m$value\e[0m" + ERROR_ALERT=1 + elif [ "$level" == "ERROR" ]; then + _logger "$prefix\e[91m$value\e[0m" + ERROR_ALERT=1 + elif [ "$level" == "WARN" ]; then + _logger "$prefix\e[93m$value\e[0m" + elif [ "$level" == "NOTICE" ]; then + _logger "$prefix$value" + elif [ "$level" == "DEBUG" ]; then + if [ "$DEBUG" == "yes" ]; then + _logger "$prefix$value" + fi + else + _logger "\e[41mLogger function called without proper loglevel.\e[0m" + _logger "$prefix$value" + fi +} + +function CheckEnvironment { + ## osync / obackup executable full path can be set here if it cannot be found on the system + if ! type -p $SUBPROGRAM.sh > /dev/null 2>&1 + then + if [ -f /usr/local/bin/$SUBPROGRAM.sh ] + then + SUBPROGRAM_EXECUTABLE=/usr/local/bin/$SUBPROGRAM.sh + else + Logger "Could not find $SUBPROGRAM.sh" "CRITICAL" + exit 1 + fi + else + SUBPROGRAM_EXECUTABLE=$(type -p $SUBPROGRAM.sh) + fi ## Check for CONF_FILE_PATH - if [ ! -d "$CONF_FILE_PATH" ] - then - Log "Cannot find conf file path $CONF_FILE_PATH" - Usage - fi + if [ ! -d "$CONF_FILE_PATH" ]; then + Logger "Cannot find conf file path $CONF_FILE_PATH" "CRITICAL" + Usage + fi } -function Batch -{ +function Batch { ## Get list of .conf files for i in $(ls $CONF_FILE_PATH/*.conf) do - if [ "$RUN" == "" ] - then + if [ "$RUN" == "" ]; then RUN="$i" else RUN=$RUN" $i" @@ -81,21 +97,20 @@ function Batch RERUNS=0 while ([ $MAX_EXECUTION_TIME -gt $SECONDS ] || [ $MAX_EXECUTION_TIME -eq 0 ]) && [ "$RUN" != "" ] && [ $MAX_RERUNS -gt $RERUNS ] do - Log "Obackup instances will be run for: $RUN" + Logger "$SUBPROGRAM instances will be run for: $RUN" "NOTICE" for i in $RUN do - $OBACKUP_EXECUTABLE "$i" $opts - if [ $? != 0 ] - then - Log "Run instance $(basename $i) failed" - if [ "RUN_AGAIN" == "" ] - then + $SUBPROGRAM_EXECUTABLE "$i" $opts & + wait $! + if [ $? != 0 ]; then + Logger "Run instance $(basename $i) failed" "ERROR" + if [ "RUN_AGAIN" == "" ]; then RUN_AGAIN="$i" else RUN_AGAIN=$RUN_AGAIN" $i" fi else - Log "Run instance $(basename $i) succeed." + Logger "Run instance $(basename $i) succeed." "NOTICE" fi done RUN="$RUN_AGAIN" @@ -104,44 +119,43 @@ function Batch done } -function Usage -{ - echo "$PROGRAM $PROGRAM_BUILD" - echo $AUTHOR - echo $CONTACT - echo "" - echo "Batch script to sequentially run obackup instances and rerun failed ones." - echo "Usage: obackup-batch.sh [OPTIONS]" - echo "" - echo "[OPTIONS]" - echo "--path=/path/to/conf Path to obackup conf files, defaults to /etc/obackup" +function Usage { + echo "$PROGRAM $PROGRAM_BUILD" + echo $AUTHOR + echo $CONTACT + echo "" + echo "Batch script to sequentially run osync or obackup instances and rerun failed ones." + echo "Usage: $SUBPROGRAM-batch.sh [OPTIONS]" + echo "" + echo "[OPTIONS]" + echo "--path=/path/to/conf Path to osync / obackup conf files, defaults to /etc/osync or /etc/obackup" echo "--max-reruns=X Number of runs max for failed instances, (defaults to 3)" - echo "--max-exec-time=X Retry failed instances only if max execution time not reached (defaults to 36000 seconds)" - echo "--no-maxtime Run obackup without honoring conf file defined timeouts" - echo "--dry Will run obackup without actually doing anything; just testing" - echo "--silent Will run obackup without any output to stdout, used for cron jobs" - echo "--verbose Increases output" - exit 128 + echo "--max-exec-time=X Retry failed instances only if max execution time not reached (defaults to 36000 seconds). Set to 0 to bypass execution time check." + echo "--no-maxtime Run osync / obackup without honoring conf file defined timeouts" + echo "--dry Will run osync / obackup without actually doing anything; just testing" + echo "--silent Will run osync / obackup without any output to stdout, used for cron jobs" + echo "--verbose Increases output" + exit 128 } -silent=0 -dry=0 -verbose=0 +_SILENT=0 +_DRY=0 +_VERBOSE=0 opts="" for i in "$@" do - case $i in - --silent) - silent=1 + case $i in + --silent) + _SILENT=1 opts=$opts" --silent" - ;; - --dry) - dry=1 + ;; + --dry) + _DRY=1 opts=$opts" --dry" - ;; - --verbose) - verbose=1 - opts=$opts" --verbose" + ;; + --verbose) + _VERBOSE=1 + opts=$opts" --verbose" ;; --no-maxtime) opts=$opts" --no-maxtime" @@ -159,12 +173,12 @@ do Usage ;; *) - Log "Unknown param '$i'" + Logger "Unknown param '$i'" "CRITICAL" Usage ;; esac done CheckEnvironment -Log "$(date) Obackup batch run" +Logger "$(date) $SUBPROGRAM batch run" "NOTICE" Batch diff --git a/obackup.sh b/obackup.sh index a28f1d1..6a47ef5 100755 --- a/obackup.sh +++ b/obackup.sh @@ -1,26 +1,41 @@ -#!/bin/bash +#!/usr/bin/env bash -###### Remote (or local) backup script for files & databases +###### Remote push/pull (or local) backup script for files & databases ###### (L) 2013-2015 by Orsiris "Ozy" de Jong (www.netpower.fr) -AUTHOR="(L) 2013-2015 by Orsiris \"Ozy\" de Jong" +PROGRAM="obackup" +AUTHOR="(L) 2013-2015 by Orsiris de Jong" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" -PROGRAM_VERSION=1.9pre -PROGRAM_BUILD=2015101301 +PROGRAM_VERSION=2.0-pre +PROGRAM_BUILD=2015111107 +IS_STABLE=no -## type doesn't work on platforms other than linux (bash). If if doesn't work, always assume output is not a zero exitcode -if ! type -p "$BASH" > /dev/null -then - echo "Please run this script only with bash shell. Tested on bash >= 3.2" - exit 127 +FUNC_BUILD=2015111102 +## BEGIN Generic functions for osync & obackup written in 2013-2015 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr + +## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode +if ! type "$BASH" > /dev/null; then + echo "Please run this script only with bash shell. Tested on bash >= 3.2" + exit 127 fi -## allow debugging from command line with preceding ocsync with DEBUG=yes -if [ ! "$DEBUG" == "yes" ] -then - DEBUG=no - SLEEP_TIME=.1 +# Environment variables +_DRYRUN=0 +_SILENT=0 + +# Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags +ERROR_ALERT=0 +WARN_ALERT=0 + + +## allow debugging from command line with _DEBUG=yes +if [ ! "$_DEBUG" == "yes" ]; then + _DEBUG=no + SLEEP_TIME=.1 + _VERBOSE=0 else - SLEEP_TIME=3 + SLEEP_TIME=1 + trap 'TrapError ${LINENO} $?' ERR + _VERBOSE=1 fi SCRIPT_PID=$$ @@ -29,1350 +44,694 @@ LOCAL_USER=$(whoami) LOCAL_HOST=$(hostname) ## Default log file until config file is loaded -if [ -w /var/log ] -then - LOG_FILE=/var/log/obackup.log +if [ -w /var/log ]; then + LOG_FILE="/var/log/$PROGRAM.log" else - LOG_FILE=./obackup.log + LOG_FILE="./$PROGRAM.log" fi -## Default directory where to store run files -if [ -w /tmp ] -then +## Default directory where to store temporary run files +if [ -w /tmp ]; then RUN_DIR=/tmp -elif [ -w /var/tmp ] -then +elif [ -w /var/tmp ]; then RUN_DIR=/var/tmp else RUN_DIR=. fi -## Working directory for partial downloads -PARTIAL_DIR=".obackup_workdir_partial" - -## Log a state message every $KEEP_LOGGING seconds. Should generally not be equal to soft or hard execution time so your log won't be unnecessary big. +## Log a state message every $KEEP_LOGGING seconds. Should not be equal to soft or hard execution time so your log will not be unnecessary big. KEEP_LOGGING=1801 -## Correct output of all system commands (language agnostic) +## Correct output of sort command (language agnostic sorting) export LC_ALL=C -## Global variables and forked command results -DATABASES_TO_BACKUP="" # Processed list of DBs that will be backed up -DATABASES_EXCLUDED_LIST="" # Processed list of DBs that won't be backed up -TOTAL_DATABASES_SIZE=0 # Total DB size of $DATABASES_TO_BACKUP -DIRECTORIES_RECURSE_TO_BACKUP="" # Processed list of recursive directories that will be backed up -DIRECTORIES_EXCLUDED_LIST="" # Processed list of recursive directorires that won't be backed up -DIRECTORIES_TO_BACKUP="" # Processed list of all directories to backup -TOTAL_FILES_SIZE=0 # Total file size of $DIRECTORIES_TO_BACKUP +# Standard alert mail body +MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." -# $RUN_DIR/obackup_remote_os_$SCRIPT_PID Result of remote OS detection -# $RUN_DIR/obackup_dblist_$SCRIPT_PID Databases list and sizes -# $RUN_DIR/obackup_dirs_recurse_list_$SCRIPT_PID Recursive directories list -# $RUN_DIR/obackup_local_sql_storage_$SCRIPT_PID Local free space for sql backup -# $RUN_DIR/obackup_local_file_storage_$SCRIPT_PID Local free space for file backup -# $RUN_DIR/obackup_fsize_$SCRIPT_PID Size of $DIRECTORIES_TO_BACKUP -# $RUN_DIR/obackup_rsync_output_$SCRIPT_PID Output of Rsync command -# $RUN_DIR/obackup_config_$SCRIPT_PID Parsed configuration file -# $RUN_DIR/obackup_run_local_$SCRIPT_PID Output of command to be run localy -# $RUN_DIR/obackup_run_remote_$SCRIPT_PID Output of command to be run remotely +# Default alert attachment filename +ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log" -ALERT_LOG_FILE=$RUN_DIR/obackup_lastlog # This is the path where to store a temporary log file to send by mail +# Set error exit code if a piped command fails + set -o pipefail + set -o errtrace -# $1 is the string to log, $2 == noprefix will remove prefix -function Log -{ - if [ "$2" != "noprefix" ] - then - local prefix="TIME: $SECONDS - " + +function Dummy { + sleep .1 +} + +function _Logger { + local svalue="${1}" # What to log to screen + local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value + echo -e "$lvalue" >> "$LOG_FILE" + + if [ $_SILENT -eq 0 ]; then + echo -e "$svalue" + fi +} + +function Logger { + local value="${1}" # Sentence to log (in double quotes) + local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL + + # Special case in daemon mode we should timestamp instead of counting seconds + if [ "$sync_on_changes" == "1" ]; then + prefix="$(date) - " else - local prefix="" + prefix="TIME: $SECONDS - " fi + # - echo -e "$prefix$1" >> "$LOG_FILE" - if [ $silent -eq 0 ] - then - echo -e "$prefix$1" + if [ "$level" == "CRITICAL" ]; then + _Logger "$prefix\e[41m$value\e[0m" "$prefix$value" + ERROR_ALERT=1 + return + elif [ "$level" == "ERROR" ]; then + _Logger "$prefix\e[91m$value\e[0m" "$prefix$value" + ERROR_ALERT=1 + return + elif [ "$level" == "WARN" ]; then + _Logger "$prefix\e[93m$value\e[0m" "$prefix$value" + WARN_ALERT=1 + return + elif [ "$level" == "NOTICE" ]; then + _Logger "$prefix$value" + return + elif [ "$level" == "DEBUG" ]; then + if [ "$_DEBUG" == "yes" ]; then + _Logger "$prefix$value" + return + fi + else + _Logger "\e[41mLogger function called without proper loglevel.\e[0m" + _Logger "$prefix$value" fi } -function LogError -{ - # \e[93m = light yellow, \e[0m = normal - Log "\e[93m$1\e[0m" - error_alert=1 -} +# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X +function KillChilds { + local pid="${1}" + local self="${2:-false}" -function LogDebug -{ - if [ "$DEBUG" == "yes" ] - then - Log "$1" - fi -} + if children="$(pgrep -P "$pid")"; then + for child in $children; do + KillChilds "$child" true + done + fi -# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X, see http://stackoverflow.com/a/32566506/2635443 -KillChilds() { - local pid="${1}" - local self="${2:-false}" - - if children="$(pgrep -P "$pid")"; then - for child in $children; do - KillChilds "$child" true - done - fi - - if [ "$self" == true ]; then - kill -s SIGTERM "$pid" || (sleep 10 && kill -9 "$pid" &) - fi -} - -function TrapError -{ - local JOB="$0" - local LINE="$1" - local CODE="${2:-1}" - if [ $silent -eq 0 ] - then - echo " /!\ Error in ${JOB}: Near line ${LINE}, exit code ${CODE}" + # Try to kill nicely, if not, wait 30 seconds to let Trap actions happen before killing + if [ "$self" == true ]; then + kill -s SIGTERM "$pid" || (sleep 30 && kill -9 "$pid" &) fi } -function TrapStop -{ - LogError " /!\ WARNING: Manual exit of backup script. Backups may be in inconsistent state." - exit 1 +function TrapError { + local job="$0" + local line="$1" + local code="${2:-1}" + if [ $_SILENT -eq 0 ]; then + echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}" + fi } -function TrapQuit -{ - # Kill all child processes - #if type -p pkill > /dev/null 2>&1 - #then - # ## Added || : to return success even if there is no child process to kill - # pkill -TERM -P $$ || : - #elif [ "$LOCAL_OS" == "msys" ] || [ "$OSTYPE" == "msys" ] - #then - ## This is not really a clean way to get child process pids, especially the tail -n +2 which resolves a strange char apparition in msys bash - # for pid in $(ps -a | awk '{$1=$1}$1' | awk '{print $1" "$2}' | grep " $$$" | awk '{print $1}' | tail -n +2) - # do - # kill -9 $pid > /dev/null 2>&1 - # done - #else - # for pid in $(ps -a --Group $$) - # do - # kill -9 $pid - # done - #fi - - if [ $error_alert -ne 0 ] - then - SendAlert - CleanUp - LogError "Backup script finished with errors." - else - CleanUp - Log "Backup script finshed." - fi - - KillChilds $$ > /dev/null 2>&1 -} - -function Spinner -{ - if [ $silent -eq 1 ] - then - return 1 +function Spinner { + if [ $_SILENT -eq 1 ]; then + return 0 fi case $toggle in 1) - echo -n $1" \ " + echo -n " \ " echo -ne "\r" toggle="2" ;; 2) - echo -n $1" | " + echo -n " | " echo -ne "\r" toggle="3" ;; 3) - echo -n $1" / " + echo -n " / " echo -ne "\r" toggle="4" ;; *) - echo -n $1" - " + echo -n " - " echo -ne "\r" toggle="1" ;; esac } -function Dummy -{ - exit 1; +function SedStripQuotes { + echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") } -function StripQuotes -{ - echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") +function StripSingleQuotes { + local string="${1}" + string="${string/#\'/}" # Remove singlequote if it begins string + string="${string/%\'/}" # Remove singlequote if it ends string + echo "$string" } -function EscapeSpaces -{ - echo $(echo "$1" | sed 's/ /\\ /g') +function StripDoubleQuotes { + local string="${1}" + string="${string/#\"/}" + string="${string/%\"/}" + echo "$string" } -function CleanUp -{ - if [ "$DEBUG" != "yes" ] - then - rm -f "$RUN_DIR/obackup_remote_os_$SCRIPT_PID" - rm -f "$RUN_DIR/obackup_dblist_$SCRIPT_PID" - rm -f "$RUN_DIR/obackup_local_sql_storage_$SCRIPT_PID" - rm -f "$RUN_DIR/obackup_local_file_storage_$SCRIPT_PID" - rm -f "$RUN_DIR/obackup_dirs_recurse_list_$SCRIPT_PID" - rm -f "$RUN_DIR/obackup_fsize_$SCRIPT_PID" - rm -f "$RUN_DIR/obackup_rsync_output_$SCRIPT_PID" - rm -f "$RUN_DIR/obackup_config_$SCRIPT_PID" - rm -f "$RUN_DIR/obackup_run_local_$SCRIPT_PID" - rm -f "$RUN_DIR/obackup_run_remote_$SCRIPT_PID" +function StripQuotes { + local string="${1}" + echo "$(StripSingleQuotes $(StripDoubleQuotes $string))" +} + +function EscapeSpaces { + local string="${1}" # String on which spaces will be escaped + echo "${string// /\ }" +} + +function IsNumeric { + eval "local value=\"${1}\"" # Needed so variable variables can be processed + + local re="^-?[0-9]+([.][0-9]+)?$" + if [[ $value =~ $re ]]; then + echo 1 + else + echo 0 fi } -function SendAlert -{ +function CleanUp { + + if [ "$_DEBUG" != "yes" ]; then + rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID" + fi +} + +function SendAlert { + + if [ "$_DEBUG" == "yes" ]; then + Logger "Debug mode, no warning email will be sent." "NOTICE" + return 0 + fi + + # + if [ "$_QUICK_SYNC" == "2" ]; then + Logger "Current task is a quicksync task. Will not send any alert." "NOTICE" + return 0 + fi + # + eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE" - MAIL_ALERT_MSG=$MAIL_ALERT_MSG$'\n\n'$( sed -n 'H; /^Obackup starting$/h; ${g;p;}' "$LOG_FILE") - if type -p mutt > /dev/null 2>&1 - then - echo -e $MAIL_ALERT_MSG | $(type -p mutt) -x -s "Backup alert for $BACKUP_ID" $DESTINATION_MAILS -a "$ALERT_LOG_FILE" - if [ $? != 0 ] - then - Log "WARNING: Cannot send alert email via $(type -p mutt) !!!" - else - Log "Sent alert mail using mutt." - fi - elif type -p mail > /dev/null 2>&1 - then - echo -e $MAIL_ALERT_MSG | $(type -p mail) -a "$ALERT_LOG_FILE" -s "Backup alert for $BACKUP_ID" $DESTINATION_MAILS - if [ $? != 0 ] - then - Log "WARNING: Cannot send alert email via $(type -p mail) with attachments !!!" - echo -e $MAIL_ALERT_MSG | $(type -p mail) -s "Backup alert for $BACKUP_ID" $DESTINATION_MAILS - if [ $? != 0 ] - then - Log "WARNING: Cannot send alert email via $(type -p mail) without attachments !!!" - else - Log "Sent alert mail using mail command without attachment." - fi - else - Log "Sent alert mail using mail command." - fi - elif type -p sendemail > /dev/null 2>&1 - then - $(type -p sendemail) -f $SENDER_MAIL -t $DESTINATION_MAILS -u "Backup alert for $BACKUP_ID" -m "$MAIL_ALERT_MSG" -s $SMTP_SERVER -o username $SMTP_USER -p password $SMTP_PASSWORD > /dev/null 2>&1 - if [ $? != 0 ] - then - Log "WARNING: Cannot send alert email via $(type -p sendemail) !!!" - else - Log "Sent alert mail using mail command without attachment." - fi - else - Log "WARNING: Cannot send alert email (no mutt / mail present) !!!" + MAIL_ALERT_MSG="$MAIL_ALERT_MSG"$'\n\n'$(tail -n 25 "$LOG_FILE") + if [ $ERROR_ALERT -eq 1 ]; then + subject="Error alert for $INSTANCE_ID" + elif [ $WARN_ALERT -eq 1 ]; then + subject="Warning alert for $INSTANCE_ID" + else + subject="Alert for $INSTANCE_ID" fi - if [ -f "$ALERT_LOG_FILE" ] - then + # Need better fallback if mail sending does not succeed + if type mutt > /dev/null 2>&1 ; then + echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS -a "$ALERT_LOG_FILE" + if [ $? != 0 ]; then + Logger "WARNING: Cannot send alert email via $(type -p mutt) !!!" "WARN" + else + Logger "Sent alert mail using mutt." "NOTICE" + return 0 + fi + fi + + if type mail > /dev/null 2>&1 ; then + echo "$MAIL_ALERT_MSG" | $(type -p mail) -a "$ALERT_LOG_FILE" -s "$subject" $DESTINATION_MAILS + if [ $? != 0 ]; then + Logger "WARNING: Cannot send alert email via $(type -p mail) with attachments !!!" "WARN" + echo "$MAIL_ALERT_MSG" | $(type -p mail) -s "$subject" $DESTINATION_MAILS + if [ $? != 0 ]; then + Logger "WARNING: Cannot send alert email via $(type -p mail) without attachments !!!" "WARN" + else + Logger "Sent alert mail using mail command without attachment." "NOTICE" + return 0 + fi + else + Logger "Sent alert mail using mail command." "NOTICE" + return 0 + fi + fi + + if type sendmail > /dev/null 2>&1 ; then + echo -e "$subject\r\n$MAIL_ALERT_MSG" | $(type -p sendmail) $DESTINATION_MAILS + if [ $? != 0 ]; then + Logger "WARNING: Cannot send alert email via $(type -p sendmail) !!!" "WARN" + else + Logger "Sent alert mail using sendmail command without attachment." "NOTICE" + return 0 + fi + fi + + if type sendemail > /dev/null 2>&1 ; then + if [ "$SMTP_USER" != "" ] && [ "$SMTP_PASSWORD" != "" ]; then + SMTP_OPTIONS="-xu $SMTP_USER -xp $SMTP_PASSWORD" + else + SMTP_OPTIONS="" + fi + $(type -p sendemail) -f $SENDER_MAIL -t $DESTINATION_MAILS -u "$subject" -m "$MAIL_ALERT_MSG" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1 + if [ $? != 0 ]; then + Logger "WARNING: Cannot send alert email via $(type -p sendemail) !!!" "WARN" + else + Logger "Sent alert mail using sendemail command without attachment." "NOTICE" + return 0 + fi + fi + + # If function has not returned 0 yet, assume it's critical that no alert can be sent + Logger "/!\ CRITICAL: Cannot send alert" "ERROR" # Is not marked critical because execution must continue + + # Delete tmp log file + if [ -f "$ALERT_LOG_FILE" ]; then rm "$ALERT_LOG_FILE" fi } -function LoadConfigFile -{ - if [ ! -f "$1" ] - then - LogError "Cannot load backup configuration file [$1]. Backup cannot start." - return 1 - elif [[ "$1" != *".conf" ]] - then - LogError "Wrong configuration file supplied [$1]. Backup cannot start." - return 1 +function LoadConfigFile { + + local config_file="${1}" + + if [ ! -f "$config_file" ]; then + Logger "Cannot load configuration file [$config_file]. Cannot start." "CRITICAL" + exit 1 + elif [[ "$1" != *".conf" ]]; then + Logger "Wrong configuration file supplied [$config_file]. Cannot start." "CRITICAL" + exit 1 else - egrep '^#|^[^ ]*=[^;&]*' "$1" > "$RUN_DIR/obackup_config_$SCRIPT_PID" - source "$RUN_DIR/obackup_config_$SCRIPT_PID" + grep '^[^ ]*=[^;&]*' "$config_file" > "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" # WITHOUT COMMENTS + # Shellcheck source=./sync.conf + source "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" fi } -function CheckEnvironment -{ - sed --version > /dev/null 2>&1 - if [ $? != 0 ] - then - LogError "GNU coreutils not found (tested for sed --version). Backup cannot start." - return 1 +function GetLocalOS { + + local local_os_var=$(uname -spio 2>&1) + if [ $? != 0 ]; then + local local_os_var=$(uname -v 2>&1) + if [ $? != 0 ]; then + local local_os_var=($uname) + fi fi + case $local_os_var in + *"Linux"*) + LOCAL_OS="Linux" + ;; + *"BSD"*) + LOCAL_OS="BSD" + ;; + *"MINGW32"*) + LOCAL_OS="msys" + ;; + *"Darwin"*) + LOCAL_OS="MacOSX" + ;; + *) + Logger "Running on >> $local_os_var << not supported. Please report to the author." "ERROR" + exit 1 + ;; + esac + Logger "Local OS: [$local_os_var]." "DEBUG" +} - if [ "$REMOTE_BACKUP" == "yes" ] - then - if ! type -p ssh > /dev/null 2>&1 - then - LogError "ssh not present. Cannot start backup." - return 1 +function GetRemoteOS { + + if [ "$REMOTE_OPERATION" == "yes" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + local cmd=$SSH_CMD' "uname -spio" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 120 240 $FUNCNAME"-1" + retval=$? + if [ $retval != 0 ]; then + local cmd=$SSH_CMD' "uname -v" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 120 240 $FUNCNAME"-2" + retval=$? + if [ $retval != 0 ]; then + local cmd=$SSH_CMD' "uname" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 120 240 $FUNCNAME"-3" + retval=$? + if [ $retval != 0 ]; then + Logger "Cannot Get remote OS type." "ERROR" + fi + fi fi - if [ "$BACKUP_SQL" != "no" ] - then - if ! type -p mysqldump > /dev/null 2>&1 - then - LogError "mysqldump not present. Cannot start backup." + local remote_os_var=$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID) + + case $remote_os_var in + *"Linux"*) + REMOTE_OS="Linux" + ;; + *"BSD"*) + REMOTE_OS="BSD" + ;; + *"MINGW32"*) + REMOTE_OS="msys" + ;; + *"Darwin"*) + REMOTE_OS="MacOSX" + ;; + *"ssh"*|*"SSH"*) + Logger "Cannot connect to remote system." "CRITICAL" + exit 1 + ;; + *) + Logger "Running on remote OS failed. Please report to the author if the OS is not supported." "CRITICAL" + Logger "Remote OS said:\n$remote_os_var" "CRITICAL" + exit 1 + esac + + Logger "Remote OS: [$remote_os_var]." "DEBUG" + fi +} + +function WaitForTaskCompletion { + local pid="${1}" # pid to wait for + local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. + local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. + local caller_name="${4}" # Who called this function + + local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once + local log_ttime=0 # local time instance for comparaison + + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function + + while eval "$PROCESS_TEST_CMD" > /dev/null + do + Spinner + exec_time=$(($SECONDS - $seconds_begin)) + if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then + if [ $log_ttime -ne $exec_time ]; then + log_ttime=$exec_time + Logger "Current task still running." "NOTICE" + fi + fi + if [ $exec_time -gt $soft_max_time ]; then + if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then + Logger "Max soft execution time exceeded for task." "WARN" + soft_alert=1 + SendAlert + + fi + if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then + Logger "Max hard execution time exceeded for task. Stopping task execution." "ERROR" + kill -s SIGTERM $pid + if [ $? == 0 ]; then + Logger "Task stopped succesfully" "NOTICE" + else + Logger "Sending SIGTERM to proces failed. Trying the hard way." "ERROR" + sleep 5 && kill -9 $pid + if [ $? != 0 ]; then + Logger "Could not stop task." "ERROR" + fi + fi return 1 fi fi - fi - - if [ "$BACKUP_FILES" != "no" ] - then - if ! type -p rsync > /dev/null 2>&1 - then - LogError "rsync not present. Backup cannot start." - return 1 - fi - fi + sleep $SLEEP_TIME + done + wait $pid + local retval=$? + return $retval } -function GetLocalOS -{ - LOCAL_OS_VAR=$(uname -spio 2>&1) - if [ $? != 0 ] - then - LOCAL_OS_VAR=$(uname -v 2>&1) - if [ $? != 0 ] - then - LOCAL_OS_VAR=($uname) - fi - fi +function WaitForCompletion { + local pid="${1}" # pid to wait for + local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. + local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. + local caller_name="${4}" # Who called this function - case $LOCAL_OS_VAR in - *"Linux"*) - LOCAL_OS="Linux" - ;; - *"BSD"*) - LOCAL_OS="BSD" - ;; - *"MINGW32"*) - LOCAL_OS="msys" - ;; - *"Darwin"*) - LOCAL_OS="MacOSX" - ;; - *) - LogError "Running on >> $LOCAL_OS_VAR << not supported. Please report to the author." - exit 1 - ;; - esac - LogDebug "Local OS: [$LOCAL_OS_VAR]." -} + local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once + local log_ttime=0 # local time instance for comparaison -function GetRemoteOS -{ - if [ "$REMOTE_SYNC" == "yes" ] - then - CheckConnectivity3rdPartyHosts - CheckConnectivityRemoteHost - eval "$SSH_CMD \"uname -spio\" > $RUN_DIR/obackup_remote_os_$SCRIPT_PID 2>&1" - child_pid=$! - WaitForTaskCompletion $child_pid 120 240 - retval=$? - if [ $retval != 0 ] - then - eval "$SSH_CMD \"uname -v\" >> $RUN_DIR/obackup_remote_os_$SCRIPT_PID 2>&1" - child_pid=$! - WaitForTaskCompletion $child_pid 120 240 - retval=$? - if [ $retval != 0 ] - then - eval "$SSH_CMD \"uname\" >> $RUN_DIR/obackup_remote_os_$SCRIPT_PID 2>&1" - child_pid=$! - WaitForTaskCompletion $child_pid 120 240 - retval=$? - if [ $retval != 0 ] - then - LogError "Cannot Get remote OS type." - fi - fi - fi + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function - REMOTE_OS_VAR=$(cat $RUN_DIR/obackup_remote_os_$SCRIPT_PID) - - case $REMOTE_OS_VAR in - *"Linux"*) - REMOTE_OS="Linux" - ;; - *"BSD"*) - REMOTE_OS="BSD" - ;; - *"MINGW32"*) - REMOTE_OS="msys" - ;; - *"Darwin"*) - REMOTE_OS="MacOSX" - ;; - *"ssh"*|*"SSH"*) - LogError "Cannot connect to remote system." - exit 1 - ;; - *) - LogError "Running on remote OS failed. Please report to the author if the OS is not supported." - LogError "Remote OS said:\n$REMOTE_OS_VAR" - exit 1 - esac - - LogDebug "Remote OS: [$REMOTE_OS_VAR]." - fi -} - -# Waits for pid $1 to complete. Will log an alert if $2 seconds passed since current task execution unless $2 equals 0. -# Will stop task and log alert if $3 seconds passed since current task execution unless $3 equals 0. -function WaitForTaskCompletion -{ - soft_alert=0 - log_time=0 - SECONDS_BEGIN=$SECONDS - while eval "$PROCESS_TEST_CMD" > /dev/null - do - Spinner - EXEC_TIME=$(($SECONDS - $SECONDS_BEGIN)) - if [ $((($EXEC_TIME + 1) % $KEEP_LOGGING)) -eq 0 ] - then - if [ $log_time -ne $EXEC_TIME ] - then - log_time=$EXEC_TIME - Log "Current task still running." + while eval "$PROCESS_TEST_CMD" > /dev/null + do + Spinner + if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ]; then + if [ $log_time -ne $SECONDS ]; then + log_time=$SECONDS + Logger "Current task still running." "NOTICE" fi - fi - if [ $EXEC_TIME -gt "$2" ] - then - if [ $soft_alert -eq 0 ] && [ "$2" != 0 ] - then - LogError "Max soft execution time exceeded for task." - soft_alert=1 - fi - if [ $EXEC_TIME -gt "$3" ] && [ "$3" != 0 ] - then - LogError "Max hard execution time exceeded for task. Stopping task execution." - kill -s SIGTERM $1 - if [ $? == 0 ] - then - LogError "Task stopped succesfully" - else - LogError "Sending SIGTERM to proces failed. Trying the hard way." - kill -9 $1 - if [ $? != 0 ] - then - LogError "Could not stop task." - fi - fi - return 1 - fi - fi - sleep $SLEEP_TIME - done - wait $child_pid - return $? + fi + if [ $SECONDS -gt $soft_max_time ]; then + if [ $soft_alert -eq 0 ] && [ $soft_max_time != 0 ]; then + Logger "Max soft execution time exceeded for script." "WARN" + soft_alert=1 + SendAlert + fi + if [ $SECONDS -gt $hard_max_time ] && [ $hard_max_time != 0 ]; then + Logger "Max hard execution time exceeded for script. Stopping current task execution." "ERROR" + kill -s SIGTERM $pid + if [ $? == 0 ]; then + Logger "Task stopped succesfully" "NOTICE" + else + Logger "Sending SIGTERM to proces failed. Trying the hard way." "ERROR" + kill -9 $pid + if [ $? != 0 ]; then + Logger "Could not stop task." "ERROR" + fi + fi + return 1 + fi + fi + sleep $SLEEP_TIME + done + wait $pid + retval=$? + return $retval } +function RunLocalCommand { + local command="${1}" # Command to run + local hard_max_time="${2}" # Max time to wait for command to compleet -## Runs local command $1 and waits for completition in $2 seconds -function RunLocalCommand -{ - if [ $dryrun -ne 0 ] - then - Log "Dryrun: Local command [$1] not run." - return 0 + if [ $_DRYRUN -ne 0 ]; then + Logger "Dryrun: Local command [$command] not run." "NOTICE" + return 1 fi - Log "Running command [$1] on local host." - eval "$1" > $RUN_DIR/obackup_run_local_$SCRIPT_PID 2>&1 & - child_pid=$! - WaitForTaskCompletion $child_pid 0 $2 + Logger "Running command [$command] on local host." "NOTICE" + eval "$command" > "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" 2>&1 & + WaitForTaskCompletion $! 0 $hard_max_time $FUNCNAME retval=$? - if [ $retval -eq 0 ] - then - Log "Command succeded." + if [ $retval -eq 0 ]; then + Logger "Command succeded." "NOTICE" else - LogError "Command failed." + Logger "Command failed." "ERROR" fi - if [ $verbose -eq 1 ] || [ $retval -ne 0 ] - then - Log "Command output:\n$(cat $RUN_DIR/obackup_run_local_$SCRIPT_PID)" + if [ $_VERBOSE -eq 1 ] || [ $retval -ne 0 ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE" fi - if [ "$STOP_ON_CMD_ERROR" == "yes" ] && [ $retval -ne 0 ] - then - exit 1 - fi - + if [ "$STOP_ON_CMD_ERROR" == "yes" ] && [ $retval -ne 0 ]; then + Logger "Stopping on command execution error." "CRITICAL" + exit 1 + fi } ## Runs remote command $1 and waits for completition in $2 seconds -function RunRemoteCommand -{ +function RunRemoteCommand { + local command="${1}" # Command to run + local hard_max_time="${2}" # Max time to wait for command to compleet + CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - if [ $dryrun -ne 0 ] - then - Log "Dryrun: Remote command [$1] not run." - return 0 - fi - Log "Running command [$1] on remote host." - eval "$SSH_CMD \"$1\" > $RUN_DIR/obackup_run_remote_$SCRIPT_PID 2>&1 &" - child_pid=$! - WaitForTaskCompletion $child_pid 0 $2 + if [ $_DRYRUN -ne 0 ]; then + Logger "Dryrun: Local command [$command] not run." "NOTICE" + return 1 + fi + Logger "Running command [$command] on remote host." "NOTICE" + cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 0 $hard_max_time $FUNCNAME retval=$? - if [ $retval -eq 0 ] - then - Log "Command succeded." + if [ $retval -eq 0 ]; then + Logger "Command succeded." "NOTICE" else - LogError "Command failed." + Logger "Command failed." "ERROR" fi - if [ -f $RUN_DIR/obackup_run_remote_$SCRIPT_PID ] && ([ $verbose -eq 1 ] || $retval -ne 0 ]) + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ] && ([ $_VERBOSE -eq 1 ] || [ $retval -ne 0 ]) then - Log "Command output:\n$(cat $RUN_DIR/obackup_run_remote_$SCRIPT_PID)" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE" fi - if [ "$STOP_ON_CMD_ERROR" == "yes" ] && [ $retval -ne 0 ] - then + if [ "$STOP_ON_CMD_ERROR" == "yes" ] && [ $retval -ne 0 ]; then + Logger "Stopping on command execution error." "CRITICAL" exit 1 - fi + fi } -function RunBeforeHook -{ - if [ "$LOCAL_RUN_BEFORE_CMD" != "" ] - then +function RunBeforeHook { + + if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE fi - if [ "$REMOTE_RUN_BEFORE_CMD" != "" ] - then + if [ "$REMOTE_RUN_BEFORE_CMD" != "" ]; then RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE fi } -function RunAfterHook -{ - if [ "$LOCAL_RUN_AFTER_CMD" != "" ] - then - RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER - fi +function RunAfterHook { - if [ "$REMOTE_RUN_AFTER_CMD" != "" ] - then + if [ "$LOCAL_RUN_AFTER_CMD" != "" ]; then + RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER + fi + + if [ "$REMOTE_RUN_AFTER_CMD" != "" ]; then RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER fi } -function CreateLocalStorageDirectories -{ - if [ ! -d $LOCAL_SQL_STORAGE ] && [ "$BACKUP_SQL" != "no" ] - then - mkdir -p $LOCAL_SQL_STORAGE - if [ $? != 0 ] - then - LogError "Cannot create directory $LOCAL_SQL_STORAGE" - return 1 - fi - fi +function CheckConnectivityRemoteHost { - if [ ! -d $LOCAL_FILE_STORAGE ] && [ "$BACKUP_FILES" != "no" ] - then - mkdir -p $LOCAL_FILE_STORAGE - if [ $? != 0 ] - then - LogError "Cannot create directory $LOCAL_FILE_STORAGE" - return 1 - fi - fi -} + if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug -function CheckSpaceRequirements -{ - if [ "$BACKUP_SQL" != "no" ] - then - if [ -w $LOCAL_SQL_STORAGE ] - then - # Not elegant solution to make df silent on errors - df -P $LOCAL_SQL_STORAGE > $RUN_DIR/obackup_local_sql_storage_$SCRIPT_PID 2>&1 - if [ $? != 0 ] - then - LOCAL_SQL_SPACE=0 - LogError "Command Output:\n$(cat $RUN_DIR/obackup_local_sql_storage_$SCRIPT_PID)" - else - LOCAL_SQL_SPACE=$(cat $RUN_DIR/obackup_local_sql_storage_$SCRIPT_PID | tail -1 | awk '{print $4}') - LOCAL_SQL_DRIVE=$(cat $RUN_DIR/obackup_local_sql_storage_$SCRIPT_PID | tail -1 | awk '{print $1}') + if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then + eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & + WaitForTaskCompletion $! 180 180 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot ping $REMOTE_HOST" "CRITICAL" + return 1 fi - - if [ $LOCAL_SQL_SPACE -eq 0 ] - then - LogError "Local sql storage space reported to be 0Ko." - elif [ $LOCAL_SQL_SPACE -lt $TOTAL_DATABASES_SIZE ] - then - LogError "Local disk space may be insufficient to backup files (available space is lower than non compressed databases)." - fi - else - LOCAL_SQL_SPACE=0 - LogError "SQL storage path [$LOCAL_SQL_STORAGE] doesn't exist or cannot write to it." fi - else - LOCAL_SQL_SPACE=0 - fi - - if [ "$BACKUP_FILES" != "no" ] - then - if [ -w $LOCAL_FILE_STORAGE ] - then - df -P $LOCAL_FILE_STORAGE > $RUN_DIR/obackup_local_file_storage_$SCRIPT_PID 2>&1 - if [ $? != 0 ] - then - LOCAL_FILE_SPACE=0 - LogError "Command Output:\n$(cat $RUN_DIR/obackup_local_file_storage_$SCRIPT_PID)" - else - LOCAL_FILE_SPACE=$(cat $RUN_DIR/obackup_local_file_storage_$SCRIPT_PID | tail -1 | awk '{print $4}') - LOCAL_FILE_DRIVE=$(cat $RUN_DIR/obackup_local_file_storage_$SCRIPT_PID | tail -1 | awk '{print $1}') - fi - - if [ $LOCAL_FILE_SPACE -eq 0 ] - then - LogError "Local file storage space reported to be 0Ko." - elif [ $LOCAL_FILE_SPACE -lt $TOTAL_FILES_SIZE ] - then - LogError "Local disk space may be insufficient to backup files (available space is lower than full backup)." - fi - else - LOCAL_FILE_SPACE=0 - LogError "File storage path [$LOCAL_FILE_STORAGE] doesn't exist or cannot write to it." - fi - else - LOCAL_FILE_SPACE=0 - fi - - if [ "$LOCAL_SQL_DRIVE" == "$LOCAL_FILE_DRIVE" ] - then - LOCAL_SPACE=$LOCAL_FILE_SPACE - else - LOCAL_SPACE=$(($LOCAL_SQL_SPACE+$LOCAL_FILE_SPACE)) - fi - - if [ $BACKUP_SIZE_MINIMUM -gt $(($TOTAL_DATABASES_SIZE+$TOTAL_FILES_SIZE)) ] && [ "$DISABLE_GET_BACKUP_FILE_SIZE" != "yes" ] - then - LogError "Backup size is smaller than expected." - fi - - if [ $LOCAL_STORAGE_WARN_MIN_SPACE -gt $LOCAL_SPACE ] - then - LogError "Local disk space is lower than warning value [$LOCAL_STORAGE_WARN_MIN_SPACE Ko]." - fi - - Log "Local Space: $LOCAL_SPACE Ko - Databases size: $TOTAL_DATABASES_SIZE Ko - Files size: $TOTAL_FILES_SIZE Ko" -} - -function CheckTotalExecutionTime -{ - #### Check if max execution time of whole script as been reached - if [ $SECONDS -gt $SOFT_MAX_EXEC_TIME_TOTAL ] - then - if [ $soft_alert_total -eq 0 ] - then - LogError "Max soft execution time of the whole backup exceeded while backing up $BACKUP_TASK." - soft_alert_total=1 - fi - if [ $SECONDS -gt $HARD_MAX_EXEC_TIME_TOTAL ] && [ $HARD_MAX_EXEC_TIME_TOTAL -ne 0 ] - then - LogError "Max hard execution time of the whole backup exceeded while backing up $BACKUP_TASK, stopping backup process." - exit 1 - fi - fi -} - -function CheckConnectivityRemoteHost -{ - if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_SYNC" != "no" ] - then - eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" - if [ $? != 0 ] - then - LogError "Cannot ping $REMOTE_HOST" - exit 1 - fi - fi -} - - -function CheckConnectivity3rdPartyHosts -{ - if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ] - then - remote_3rd_party_success=0 - OLD_IFS=$IFS - IFS=$' \t\n' - for i in $REMOTE_3RD_PARTY_HOSTS - do - eval "$PING_CMD $i > /dev/null 2>&1" - if [ $? != 0 ] - then - Log "Cannot ping 3rd party host $i" - else - remote_3rd_party_success=1 - fi - done - IFS=$OLD_IFS - if [ $remote_3rd_party_success -ne 1 ] - then - LogError "No remote 3rd party host responded to ping. No internet ?" - exit 1 - fi - fi -} - - -function ListDatabases -{ - SECONDS_BEGIN=$SECONDS - Log "Listing databases." - CheckConnectivity3rdPartyHosts - CheckConnectivityRemoteHost - if [ "$REMOTE_BACKUP" != "no" ] - then - sql_cmd="$SSH_CMD \"mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > $RUN_DIR/obackup_dblist_$SCRIPT_PID &" - else - sql_cmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/obackup_dblist_$SCRIPT_PID &" - fi - - LogDebug "$sql_cmd" - eval "$sql_cmd 2>&1" - child_pid=$! - WaitForTaskCompletion $child_pid $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK - retval=$? - if [ $retval -eq 0 ] - then - Log "Listing databases succeeded." - else - LogError "Listing databases failed." - if [ -f $RUN_DIR/obackup_dblist_$SCRIPT_PID ] - then - LogError "Command output:\n$(cat $RUN_DIR/obackup_dblist_$SCRIPT_PID)" - fi - return $retval - fi - - OLD_IFS=$IFS - IFS=$' \n' - for line in $(cat $RUN_DIR/obackup_dblist_$SCRIPT_PID) - do - db_name=$(echo $line | cut -f1) - db_size=$(echo $line | cut -f2) - - if [ "$DATABASES_ALL" == "yes" ] - then - db_backup=1 - for j in $DATABASES_ALL_EXCLUDE_LIST - do - if [ "$db_name" == "$j" ] - then - db_backup=0 - fi - done - else - db_backup=0 - for j in $DATABASES_LIST - do - if [ "$db_name" == "$j" ] - then - db_backup=1 - fi - done - fi - - if [ $db_backup -eq 1 ] - then - if [ "$DATABASES_TO_BACKUP" != "" ] - then - DATABASES_TO_BACKUP="$DATABASES_TO_BACKUP $db_name" - else - DATABASES_TO_BACKUP=$db_name - fi - TOTAL_DATABASES_SIZE=$((TOTAL_DATABASES_SIZE+$db_size)) - else - DATABASES_EXCLUDED_LIST="$DATABASES_EXCLUDED_LIST $db_name" - fi - done - IFS=$OLD_IFS - - if [ $verbose -eq 1 ] - then - Log "Database backup list: $DATABASES_TO_BACKUP" - Log "Database exclude list: $DATABASES_EXCLUDED_LIST" fi } -function BackupDatabase -{ - CheckConnectivity3rdPartyHosts - if [ "$REMOTE_BACKUP" == "yes" ] && [ "$COMPRESSION_REMOTE" == "no" ] - then - CheckConnectivityRemoteHost - if [ $? != 0 ] - then - LogError "Connectivity test failed. Stopping current task." - exit 1 - fi - dry_sql_cmd="$SSH_CMD mysqldump -u $SQL_USER --skip-lock-tables --single-transaction --database $1 2>&1 > /dev/null" - sql_cmd="$SSH_CMD mysqldump -u $SQL_USER --skip-lock-tables --single-transaction --database $1 $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS > $LOCAL_SQL_STORAGE/$1.sql$COMPRESSION_EXTENSION" - elif [ "$REMOTE_BACKUP" == "yes" ] && [ "$COMPRESSION_REMOTE" == "yes" ] - then - CheckConnectivityRemoteHost - if [ $? != 0 ] - then - LogError "Connectivity test failed. Stopping current task." - exit 1 - fi - dry_sql_cmd="$SSH_CMD \"mysqldump -u $SQL_USER --skip-lock-tables --single-transaction --database $1 $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS\" 2>&1 > /dev/null" - sql_cmd="$SSH_CMD \"mysqldump -u $SQL_USER --skip-lock-tables --single-transaction --database $1 $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS\" > $LOCAL_SQL_STORAGE/$1.sql$COMPRESSION_EXTENSION" - else - dry_sql_cmd="mysqldump -u $SQL_USER --skip-lock-tables --single-transaction --database $1 $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS 2>&1 > /dev/null" - sql_cmd="mysqldump -u $SQL_USER --skip-lock-tables --single-transaction --database $1 $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS > $LOCAL_SQL_STORAGE/$1.sql$COMPRESSION_EXTENSION" - fi +function CheckConnectivity3rdPartyHosts { - if [ $dryrun -ne 1 ] - then - LogDebug "SQL_CMD: $sql_cmd" - eval "$sql_cmd 2>&1" - else - LogDebug "SQL_CMD: $dry_sql_cmd" - eval "$dry_sql_cmd" - fi - exit $? -} + if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug -function BackupDatabases -{ - OLD_IFS=$IFS - IFS=$' \t\n' - for BACKUP_TASK in $DATABASES_TO_BACKUP - do - Log "Backing up database $BACKUP_TASK" - SECONDS_BEGIN=$SECONDS - BackupDatabase $BACKUP_TASK & - child_pid=$! - WaitForTaskCompletion $child_pid $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK - retval=$? - if [ $retval -ne 0 ] - then - LogError "Backup failed." - else - Log "Backup succeeded." - fi - - CheckTotalExecutionTime - done - IFS=$OLD_IFS -} - -# Fetches single quoted directory listing including recursive ones separated by commas (eg '/dir1';'/dir2';'/dir3') -function ListDirectories -{ - SECONDS_BEGIN=$SECONDS - Log "Listing directories to backup." - OLD_IFS=$IFS - IFS=$PATH_SEPARATOR_CHAR - for dir in $DIRECTORIES_RECURSE_LIST - do - CheckConnectivity3rdPartyHosts - if [ "$REMOTE_BACKUP" == "yes" ] - then - CheckConnectivityRemoteHost - if [ $? != 0 ] - then - LogError "Connectivity test failed. Stopping current task." - Dummy & - else - eval "$SSH_CMD \"$COMMAND_SUDO $FIND_CMD -L $dir/ -mindepth 1 -maxdepth 1 -type d\" > $RUN_DIR/obackup_dirs_recurse_list_$SCRIPT_PID &" - fi - else - eval "$COMMAND_SUDO $FIND_CMD -L $dir/ -mindepth 1 -maxdepth 1 -type d > $RUN_DIR/obackup_dirs_recurse_list_$SCRIPT_PID &" - fi - child_pid=$! - WaitForTaskCompletion $child_pid $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK - retval=$? - if [ $retval != 0 ] - then - LogError "Could not enumerate recursive directories in $dir." - if [ -f $RUN_DIR/obackup_dirs_recurse_list_$SCRIPT_PID ] - then - LogError "Command output:\n$(cat $RUN_DIR/obackup_dirs_recurse_list_$SCRIPT_PID)" - fi - return 1 - fi - - OLD_IFS=$IFS - IFS=$' \n' - for line in $(cat $RUN_DIR/obackup_dirs_recurse_list_$SCRIPT_PID) - do - file_exclude=0 - IFS=$PATH_SEPARATOR_CHAR - for k in $DIRECTORIES_RECURSE_EXCLUDE_LIST + if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then + remote_3rd_party_success=0 + OLD_IFS=$IFS + IFS=$' \t\n' + for i in $REMOTE_3RD_PARTY_HOSTS do - if [ "$k" == "$line" ] - then - file_exclude=1 + eval "$PING_CMD $i > /dev/null 2>&1" & + WaitForTaskCompletion $! 360 360 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot ping 3rd party host $i" "WARN" + else + remote_3rd_party_success=1 fi done - IFS=$' \n' - - if [ $file_exclude -eq 0 ] - then - if [ "$DIRECTORIES_TO_BACKUP" == "" ] - then - DIRECTORIES_TO_BACKUP="'$line'" - else - DIRECTORIES_TO_BACKUP="$DIRECTORIES_TO_BACKUP$PATH_SEPARATOR_CHAR'$line'" - fi - else - DIRECTORIES_EXCLUDED_LIST="$DIRECTORIES_EXCLUDED_LIST$PATH_SEPARATOR_CHAR'$line'" + IFS=$OLD_IFS + if [ $remote_3rd_party_success -ne 1 ]; then + Logger "No remote 3rd party host responded to ping. No internet ?" "CRITICAL" + return 1 fi - done - Log "Listing of recursive directories succeeded for $dir" - if [ $verbose -eq 1 ] - then - Log "\n$(cat $RUN_DIR/obackup_dirs_recurse_list_$SCRIPT_PID)" fi - IFS=$OLD_IFS - done - DIRECTORIES_TO_BACKUP_RECURSE=$DIRECTORIES_TO_BACKUP - - for dir in $DIRECTORIES_SIMPLE_LIST - do - if [ "$DIRECTORIES_TO_BACKUP" == "" ] - then - DIRECTORIES_TO_BACKUP="'$dir'" - else - DIRECTORIES_TO_BACKUP="$DIRECTORIES_TO_BACKUP$PATH_SEPARATOR_CHAR'$dir'" - fi - done - - IFS=$OLD_IFS -} - -function GetDirectoriesSize -{ - # remove the path separator char from the dir list with sed 's/;/ /g' - dir_list=$(echo $DIRECTORIES_TO_BACKUP | sed 's/'"$PATH_SEPARATOR_CHAR"'/ /g' ) - Log "Getting files size" - CheckConnectivity3rdPartyHosts - if [ "$REMOTE_BACKUP" == "yes" ] - then - CheckConnectivityRemoteHost - if [ $? != 0 ] - then - LogError "Connectivity test failed. Stopping current task." - Dummy & - else - eval "$SSH_CMD \"echo $dir_list | xargs $COMMAND_SUDO du -cs | tail -n1 | cut -f1\" > $RUN_DIR/obackup_fsize_$SCRIPT_PID &" - fi - else - echo $dir_list | xargs $COMMAND_SUDO du -cs | tail -n1 | cut -f1 > $RUN_DIR/obackup_fsize_$SCRIPT_PID & - fi - child_pid=$! - WaitForTaskCompletion $child_pid $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK - retval=$? - if [ $retval != 0 ] - then - LogError "Could not get files size." - if [ -f $RUN_DIR/obackup_fsize_$SCRIPT_PID ] - then - LogError "Command output:\n$(cat $RUN_DIR/obackup_fsize_$SCRIPT_PID)" - fi - return 1 - else - Log "File size fetched successfully." - TOTAL_FILES_SIZE=$(cat $RUN_DIR/obackup_fsize_$SCRIPT_PID) fi } -function RsyncExcludePattern -{ - # Disable globbing so wildcards from exclusions don't get expanded - set -f - rest="$RSYNC_EXCLUDE_PATTERN" - while [ -n "$rest" ] - do - # Take the string until first occurence until $PATH_SEPARATOR_CHAR - str=${rest%%;*} - # Handle the last case - if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ] - then - rest= - else - # Cut everything before the first occurence of $PATH_SEPARATOR_CHAR - rest=${rest#*$PATH_SEPARATOR_CHAR} - fi +#__BEGIN_WITH_PARANOIA_DEBUG +#__END_WITH_PARANOIA_DEBUG - if [ "$RSYNC_EXCLUDE" == "" ] - then - RSYNC_EXCLUDE="--exclude=\"$str\"" - else - RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$str\"" - fi - done - set +f -} +function PreInit { -function RsyncExcludeFrom -{ - if [ ! $RSYNC_EXCLUDE_FROM == "" ] - then - ## Check if the exclude list has a full path, and if not, add the config file path if there is one - if [ "$(basename $RSYNC_EXCLUDE_FROM)" == "$RSYNC_EXCLUDE_FROM" ] - then - RSYNC_EXCLUDE_FROM=$(dirname $ConfigFile)/$RSYNC_EXCLUDE_FROM - fi - - if [ -e $RSYNC_EXCLUDE_FROM ] - then - RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\"" - fi - fi -} - -function Rsync -{ - i="$(StripQuotes $1)" - if [ "$LOCAL_STORAGE_KEEP_ABSOLUTE_PATHS" == "yes" ] - then - local_file_storage_path="$(dirname $LOCAL_FILE_STORAGE$i)" - else - local_file_storage_path=$LOCAL_FILE_STORAGE - fi - - ## Manage to backup recursive directories lists files only (not recursing into subdirectories) - if [ "$2" == "no-recurse" ] - then - RSYNC_EXCLUDE=$RSYNC_EXCLUDE" --exclude=*/*/" - # Fixes symlinks to directories in target cannot be deleted when backing up root directory without recursion - RSYNC_NO_RECURSE_ARGS=" -k" - else - RSYNC_NO_RECURSE_ARGS="" - fi - - # Creating subdirectories because rsync cannot handle mkdir -p - if [ ! -d $local_file_storage_path/$1 ] - then - mkdir -p "$local_file_storage_path/$1" - fi - - CheckConnectivity3rdPartyHosts - if [ "$REMOTE_BACKUP" == "yes" ] - then - CheckConnectivityRemoteHost - if [ $? != 0 ] - then - LogError "Connectivity test failed. Stopping current task." - exit 1 - fi - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$1\" \"$local_file_storage_path\" > $RUN_DIR/obackup_rsync_output_$SCRIPT_PID 2>&1" - else - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$1\" \"$local_file_storage_path\" > $RUN_DIR/obackup_rsync_output_$SCRIPT_PID 2>&1" - fi - #### Eval is used so the full command is processed without bash adding single quotes round variables - LogDebug "RSYNC_CMD: $rsync_cmd" - - eval $rsync_cmd - exit $? -} - -#### First backup simple list then recursive list -function FilesBackup -{ - OLD_IFS=$IFS - IFS=$PATH_SEPARATOR_CHAR - for BACKUP_TASK in $DIRECTORIES_SIMPLE_LIST - do - BACKUP_TASK=$(StripQuotes $BACKUP_TASK) - Log "Beginning recursive file backup on $BACKUP_TASK" - SECONDS_BEGIN=$SECONDS - Rsync $BACKUP_TASK & - child_pid=$! - WaitForTaskCompletion $child_pid $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK - retval=$? - if [ $verbose -eq 1 ] && [ -f $RUN_DIR/obackup_rsync_output_$SCRIPT_PID ] - then - Log "List:\n$(cat $RUN_DIR/obackup_rsync_output_$SCRIPT_PID)" - fi - - if [ $retval -ne 0 ] - then - LogError "Backup failed on remote files." - if [ $verbose -eq 0 ] && [ -f $RUN_DIR/obackup_rsync_output_$SCRIPT_PID ] - then - LogError "$(cat $RUN_DIR/obackup_rsync_output_$SCRIPT_PID)" - fi - else - Log "Backup succeeded." - fi - CheckTotalExecutionTime - done - - ## Also backup files at root of DIRECTORIES_RECURSE_LIST directories - for BACKUP_TASK in $DIRECTORIES_RECURSE_LIST - do - BACKUP_TASK="$(StripQuotes $BACKUP_TASK)" - Log "Beginning non recursive file backup on $BACKUP_TASK" - SECONDS_BEGIN=$SECONDS - Rsync $BACKUP_TASK "no-recurse" & - child_pid=$! - WaitForTaskCompletion $child_pid $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK - retval=$? - if [ $verbose -eq 1 ] && [ -f $RUN_DIR/obackup_rsync_output_$SCRIPT_PID ] - then - Log "List:\n$(cat $RUN_DIR/obackup_rsync_output_$SCRIPT_PID)" - fi - - if [ $retval -ne 0 ] - then - LogError "Backup failed on remote files." - if [ $verbose -eq 0 ] && [ -f $RUN_DIR/obackup_rsync_output_$SCRIPT_PID ] - then - LogError "$(cat $RUN_DIR/obackup_rsync_output_$SCRIPT_PID)" - fi - else - Log "Backup succeeded." - fi - CheckTotalExecutionTime - done - - for BACKUP_TASK in $DIRECTORIES_TO_BACKUP_RECURSE - do - BACKUP_TASK=$(StripQuotes $BACKUP_TASK) - Log "Beginning recursive file backup on $BACKUP_TASK" - SECONDS_BEGIN=$SECONDS - Rsync $BACKUP_TASK "recurse" & - child_pid=$! - WaitForTaskCompletion $child_pid $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK - retval=$? - if [ $verbose -eq 1 ] && [ -f $RUN_DIR/obackup_rsync_output_$SCRIPT_PID ] - then - Log "List:\n$(cat $RUN_DIR/obackup_rsync_output_$SCRIPT_PID)" - fi - - if [ $retval -ne 0 ] - then - LogError "Backup failed on remote files." - if [ $verbose -eq 0 ] && [ -f $RUN_DIR/obackup_rsync_output_$SCRIPT_PID ] - then - LogError "$(cat $RUN_DIR/obackup_rsync_output_$SCRIPT_PID)" - fi - else - Log "Backup succeeded." - fi - CheckTotalExecutionTime - done - - IFS=$OLD_IFS -} - -# Will rotate everything in $1 -function RotateBackups -{ - OLD_IFS=$IFS - IFS=$' \t\n' - for backup in $(ls -I "*.obackup.*" $1) - do - copy=$ROTATE_COPIES - while [ $copy -gt 1 ] - do - if [ $copy -eq $ROTATE_COPIES ] - then - rm -rf "$1/$backup.obackup.$copy" & - child_pid=$! - WaitForTaskCompletion $child_pid 0 0 - fi - path="$1/$backup.obackup.$(($copy-1))" - if [[ -f $path || -d $path ]] - then - mv $path "$1/$backup.obackup.$copy" & - child_pid=$! - WaitForTaskCompletion $child_pid 0 0 - - fi - copy=$(($copy-1)) - done - - # Latest file backup will not be moved if script configured for remote backup so next rsync execution will only do delta copy instead of full one - if [[ $backup == *.sql.* ]] - then - mv "$1/$backup" "$1/$backup.obackup.1" & - child_pid=$! - WaitForTaskCompletion $child_pid 0 0 - - elif [ "$REMOTE_BACKUP" == "yes" ] - then - cp -R "$1/$backup" "$1/$backup.obackup.1" & - child_pid=$! - WaitForTaskCompletion $child_pid 0 0 - - else - mv "$1/$backup" "$1/$backup.obackup.1" & - child_pid=$! - WaitForTaskCompletion $child_pid 0 0 - - fi - done - IFS=$OLD_IFS -} - -function Init -{ - # Set error exit code if a piped command fails - set -o pipefail - set -o errtrace - - trap TrapStop SIGINT SIGQUIT SIGKILL SIGTERM SIGHUP - trap TrapQuit EXIT - if [ "$DEBUG" == "yes" ] - then - trap 'TrapError ${LINENO} $?' ERR - fi - - MAIL_ALERT_MSG="Warning: Execution of obackup instance $BACKUP_ID (pid $SCRIPT_PID) as $LOCAL_USER@$LOCAL_HOST produced errors on $(date)." - - ## Set SSH command - if [ "$SSH_COMPRESSION" == "yes" ] - then + ## SSH compression + if [ "$SSH_COMPRESSION" != "no" ]; then SSH_COMP=-C else SSH_COMP= fi - SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" - RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -p $REMOTE_PORT" ## Support for older config files without RSYNC_EXECUTABLE option - if [ "$RSYNC_EXECUTABLE" == "" ] - then + if [ "$RSYNC_EXECUTABLE" == "" ]; then RSYNC_EXECUTABLE=rsync fi - ## Sudo execution option - if [ "$SUDO_EXEC" == "yes" ] - then - if [ "$RSYNC_REMOTE_PATH" != "" ] - then - RSYNC_PATH="sudo $RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" - else - RSYNC_PATH="sudo $RSYNC_EXECUTABLE" + ## Sudo execution option + if [ "$SUDO_EXEC" == "yes" ]; then + if [ "$RSYNC_REMOTE_PATH" != "" ]; then + RSYNC_PATH="sudo $RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" + else + RSYNC_PATH="sudo $RSYNC_EXECUTABLE" fi - COMMAND_SUDO="sudo" + COMMAND_SUDO="sudo" else - if [ "$RSYNC_REMOTE_PATH" != "" ] - then - RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" - else - RSYNC_PATH="$RSYNC_EXECUTABLE" - fi + if [ "$RSYNC_REMOTE_PATH" != "" ]; then + RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" + else + RSYNC_PATH="$RSYNC_EXECUTABLE" + fi COMMAND_SUDO="" fi - ## Set Rsync arguments - RSYNC_ARGS=-rlptgoDu - if [ "$PRESERVE_ACL" == "yes" ] - then - RSYNC_ARGS=$RSYNC_ARGS"A" + ## Set rsync default arguments + RSYNC_ARGS="-rlptgoD" + + if [ "$PRESERVE_ACL" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -A" fi - if [ "$PRESERVE_XATTR" == "yes" ] - then - RSYNC_ARGS=$RSYNC_ARGS"X" + if [ "$PRESERVE_XATTR" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -X" fi - if [ "$RSYNC_COMPRESS" == "yes" ] - then - RSYNC_ARGS=$RSYNC_ARGS"z" + if [ "$RSYNC_COMPRESS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -z" fi - if [ "$COPY_SYMLINKS" != "no" ] - then + if [ "$COPY_SYMLINKS" == "yes" ]; then RSYNC_ARGS=$RSYNC_ARGS" -L" fi - if [ "$KEEP_DIRLINKS" != "no" ] - then + if [ "$KEEP_DIRLINKS" == "yes" ]; then RSYNC_ARGS=$RSYNC_ARGS" -K" fi - if [ "$PRESERVE_HARDLINKS" == "yes" ] - then + if [ "$PRESERVE_HARDLINKS" == "yes" ]; then RSYNC_ARGS=$RSYNC_ARGS" -H" fi - if [ $verbose -eq 1 ] - then - RSYNC_ARGS=$RSYNC_ARGS"i" - fi - if [ $dryrun -eq 1 ] - then - RSYNC_ARGS=$RSYNC_ARGS"n" + if [ "$CHECKSUM" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --checksum" + fi + if [ $_DRYRUN -eq 1 ]; then + RSYNC_ARGS=$RSYNC_ARGS" -n" DRY_WARNING="/!\ DRY RUN" fi - if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ] - then + if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" fi - if [ "$PARTIAL" == "yes" ] - then - RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" - RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$PARTIAL_DIR\"" - fi - - if [ "$DELETE_VANISHED_FILES" == "yes" ] - then - RSYNC_ARGS=$RSYNC_ARGS" --delete" - fi - - if [ "$DELTA_COPIES" != "no" ] - then - RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file" - else - RSYNC_ARGS=$RSYNC_ARGS" --whole-file" - fi - - if [ $stats -eq 1 ] - then - RSYNC_ARGS=$RSYNC_ARGS" --stats" - fi - - ## Fix for symlink to directories on target can't get updated - RSYNC_ARGS=$RSYNC_ARGS" --force" - - ## Set compression executable and extension - if [ "$COMPRESSION_LEVEL" == "" ] - then - COMPRESSION_LEVEL=3 - fi - if type -p xz > /dev/null 2>&1 + ## Set compression executable and extension + COMPRESSION_LEVEL=3 + if type xz > /dev/null 2>&1 then COMPRESSION_PROGRAM="| xz -$COMPRESSION_LEVEL" COMPRESSION_EXTENSION=.xz - elif type -p lzma > /dev/null 2>&1 + elif type lzma > /dev/null 2>&1 then COMPRESSION_PROGRAM="| lzma -$COMPRESSION_LEVEL" COMPRESSION_EXTENSION=.lzma - elif type -p pigz > /dev/null 2>&1 + elif type pigz > /dev/null 2>&1 then COMPRESSION_PROGRAM="| pigz -$COMPRESSION_LEVEL" COMPRESSION_EXTENSION=.gz COMPRESSION_OPTIONS=--rsyncable - elif type -p gzip > /dev/null 2>&1 + elif type gzip > /dev/null 2>&1 then COMPRESSION_PROGRAM="| gzip -$COMPRESSION_LEVEL" COMPRESSION_EXTENSION=.gz @@ -1381,145 +740,1278 @@ function Init COMPRESSION_PROGRAM= COMPRESSION_EXTENSION= fi - ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" + ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" } -function InitLocalOSSettings -{ - ## If running under Msys, some commands don't run the same way +function PostInit { + + # Define remote commands + SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" + SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" + RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -p $REMOTE_PORT" +} + +function InitLocalOSSettings { + + ## If running under Msys, some commands do not run the same way ## Using mingw version of find instead of windows one ## Getting running processes is quite different - ## Ping command isn't the same - if [ "$LOCAL_OS" == "msys" ] - then + ## Ping command is not the same + if [ "$LOCAL_OS" == "msys" ]; then FIND_CMD=$(dirname $BASH)/find - ## TODO: The following command needs to be checked on msys. Does the $1 variable substitution work ? - PROCESS_TEST_CMD='ps -a | awk "{\$1=\$1}\$1" | awk "{print \$1}" | grep $1' + #TODO: The following command needs to be checked on msys. Does the $1 variable substitution work ? + # PROCESS_TEST_CMD assumes there is a variable $pid + PROCESS_TEST_CMD='ps -a | awk "{\$1=\$1}\$1" | awk "{print \$1}" | grep $pid' PING_CMD="ping -n 2" else FIND_CMD=find - PROCESS_TEST_CMD='ps -p$1' + # PROCESS_TEST_CMD assumes there is a variable $pid + PROCESS_TEST_CMD='ps -p$pid' PING_CMD="ping -c 2 -i .2" fi ## Stat command has different syntax on Linux and FreeBSD/MacOSX - if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ] - then + if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then STAT_CMD="stat -f \"%Sm\"" else STAT_CMD="stat --format %y" fi } -function InitRemoteOSSettings -{ - ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability - if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] - then +function InitRemoteOSSettings { + + ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then RSYNC_ARGS=$RSYNC_ARGS" -E" fi - if [ "$REMOTE_OS" == "msys" ] - then + if [ "$REMOTE_OS" == "msys" ]; then REMOTE_FIND_CMD=$(dirname $BASH)/find else REMOTE_FIND_CMD=find fi } -function Main -{ - if [ "$BACKUP_SQL" != "no" ] - then - ListDatabases +## END Generic functions + +## Working directory for partial downloads +PARTIAL_DIR=".obackup_workdir_partial" + +# List of runtime created global variables +# $SQL_DISK_SPACE, disk space available on target for sql backups +# $FILE_DISK_SPACE, disk space available on target for file backups +# $SQL_BACKUP_TASKS, list of all databases to backup, space separated +# $SQL_EXCLUDED_TASKS, list of all database to exclude from backup, space separated +# $FILE_BACKUP_TASKS list of directories to backup, found in config file +# $FILE_RECURSIVE_BACKUP_TASKS, list of directories to backup, computed from config file recursive list +# $FILE_RECURSIVE_EXCLUDED_TASKS, list of all directories excluded from recursive list +# $FILE_SIZE_LIST, list of all directories to include in GetDirectoriesSize + +CAN_BACKUP_SQL=1 +CAN_BACKUP_FILES=1 + +function TrapStop { + Logger " /!\ WARNING: Manual exit of backup script. Backups may be in inconsistent state." "WARN" + exit 1 +} + +function TrapQuit { + if [ $ERROR_ALERT -ne 0 ]; then + SendAlert + CleanUp + Logger "Backup script finished with errors." "ERROR" + elif [ $WARN_ALERT -ne 0 ]; then + SendAlert + CleanUp + Logger "Backup script finished with warnings." "WARN" + else + CleanUp + Logger "Backup script finshed." "NOTICE" fi - if [ "$BACKUP_FILES" != "no" ] - then - ListDirectories - if [ "$DISABLE_GET_BACKUP_FILE_SIZE" != "yes" ] - then - GetDirectoriesSize + + KillChilds $$ > /dev/null 2>&1 +} + +function CheckEnvironment { + + if [ "$REMOTE_OPERATION" == "yes" ]; then + if ! type ssh > /dev/null 2>&1 ; then + Logger "ssh not present. Cannot start backup." "CRITICAL" + exit 1 + fi + + if [ "$SQL_BACKUP" != "no" ]; then + if ! type mysqldump > /dev/null 2>&1 ; then + Logger "mysqldump not present. Cannot backup SQL." "CRITICAL" + CAN_BACKUP_SQL=0 + fi + if ! type mysql > /dev/null 2>&1 ; then + Logger "mysql not present. Cannot backup SQL." "CRITICAL" + CAN_BACKUP_SQL=0 + fi fi fi - if [ $dryrun -ne 1 ] - then - CreateLocalStorageDirectories + + if [ "$FILE_BACKUP" != "no" ]; then + if [ "$ENCRYPTION" == "yes" ]; then + if ! type duplicity > /dev/null 2>&1 ; then + Logger "duplicity not present. Cannot backup encrypted files." "CRITICAL" + CAN_BACKUP_FILES=0 + fi + else + if ! type rsync > /dev/null 2>&1 ; then + Logger "rsync not present. Cannot backup files." "CRITICAL" + CAN_BACKUP_FILES=0 + fi + fi + fi +} + +function CheckCurrentConfig { + + if [ "$INSTANCE_ID" == "" ]; then + Logger "No INSTANCE_ID defined in config file." "CRITICAL" + exit 1 fi - CheckSpaceRequirements + # Check all variables that should contain "yes" or "no" + declare -a yes_no_vars=(SQL_BACKUP FILE_BACKUP ENCRYPTION CREATE_DIRS KEEP_ABSOLUTE_PATHS GET_BACKUP_SIZE SUDO_EXEC SSH_COMPRESSION REMOTE_HOST_PING DATABASES_ALL PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS RSYNC_COMPRESS PARTIAL DELETE_VANISHED_FILES DELTA_COPIES ROTATE_SQL_BACKUPS ROTATE_FILE_BACKUPS STOP_ON_CMD_ERROR) + for i in ${yes_no_vars[@]}; do + test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file.\" \"CRITICAL\"; exit 1; fi" + eval "$test" + done + + if [ "$BACKUP_TYPE" != "local" ] && [ "$BACKUP_TYPE" != "pull" ] && [ "$BACKUP_TYPE" != "push" ]; then + Logger "Bogus BACKUP_TYPE value in config file." "CRITICAL" + exit 1 + fi + + # Check all variables that should contain a numerical value >= 0 + declare -a num_vars=(BACKUP_SIZE_MINIMUM BANDWIDTH SQL_WARN_MIN_SPACE FILE_WARN_MIN_SPACE SOFT_MAX_EXEC_TIME_DB_TASK HARD_MAX_EXEC_TIME_DB_TASK COMPRESSION_LEVEL SOFT_MAX_EXEC_TIME_FILE_TASK HARD_MAX_EXEC_TIME_FILE_TASK SOFT_MAX_EXEC_TIME_TOTAL HARD_MAX_EXEC_TIME_TOTAL ROTATE_COPIES MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER) + for i in ${num_vars[@]}; do + test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file.\" \"CRITICAL\"; exit 1; fi" + eval "$test" + done + + #TODO-v2.1: Add runtime variable tests (RSYNC_ARGS etc) +} + +function _ListDatabasesLocal { + + sql_cmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + Logger "cmd: $sql_cmd" "DEBUG" + eval "$sql_cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + if [ $? -eq 0 ]; then + Logger "Listing databases succeeded." "NOTICE" + else + Logger "Listing databases failed." "ERROR" + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + return 1 + fi + +} + +function _ListDatabasesRemote { + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + sql_cmd="$SSH_CMD \"mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > \"$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID\" 2>&1" + Logger "cmd: $sql_cmd" "DEBUG" + eval "$sql_cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + if [ $? -eq 0 ]; then + Logger "Listing databases succeeded." "NOTICE" + else + Logger "Listing databases failed." "ERROR" + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + return 1 + fi +} + +function ListDatabases { + + local output_file # Return of subfunction + + if [ $CAN_BACKUP_SQL -ne 1 ]; then + Logger "Cannot list databases." "ERROR" + return 1 + fi + + Logger "Listing databases." "NOTICE" + + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]; then + _ListDatabasesLocal + if [ $? != 0 ]; then + output_file="" + else + output_file="$RUN_DIR/$PROGRAM._ListDatabasesLocal.$SCRIPT_PID" + fi + elif [ "$BACKUP_TYPE" == "pull" ]; then + _ListDatabasesRemote + if [ $? != 0 ]; then + output_file="" + else + output_file="$RUN_DIR/$PROGRAM._ListDatabasesRemote.$SCRIPT_PID" + fi + fi + + if [ -f "$output_file" ] && [ $CAN_BACKUP_SQL -eq 1 ]; then + OLD_IFS=$IFS + IFS=$' \n' + for line in $(cat "$output_file") + do + db_name=$(echo $line | cut -f1) + db_size=$(echo $line | cut -f2) + + if [ "$DATABASES_ALL" == "yes" ]; then + db_backup=1 + IFS=$PATH_SEPARATOR_CHAR + for j in $DATABASES_ALL_EXCLUDE_LIST + do + if [ "$db_name" == "$j" ]; then + db_backup=0 + fi + done + IFS=$' \n' + else + db_backup=0 + IFS=$PATH_SEPARATOR_CHAR + for j in $DATABASES_LIST + do + if [ "$db_name" == "$j" ]; then + db_backup=1 + fi + done + IFS=$' \n' + fi + + if [ $db_backup -eq 1 ]; then + if [ "$SQL_BACKUP_TASKS" != "" ]; then + SQL_BACKUP_TASKS="$SQL_BACKUP_TASKS $db_name" + else + SQL_BACKUP_TASKS="$db_name" + fi + TOTAL_DATABASES_SIZE=$((TOTAL_DATABASES_SIZE+$db_size)) + else + SQL_EXCLUDED_TASKS="$SQL_EXCLUDED_TASKS $db_name" + fi + done + IFS=$OLD_IFS + + Logger "Database backup list: $SQL_BACKUP_TASKS" "DEBUG" + Logger "Database exclude list: $SQL_EXCLUDED_TASKS" "DEBUG" + else + Logger "Will not execute database backup." "ERROR" + CAN_BACKUP_SQL=0 + fi +} + +function _ListRecursiveBackupDirectoriesLocal { + + OLD_IFS=$IFS + IFS=$PATH_SEPARATOR_CHAR + for directory in $RECURSIVE_DIRECTORY_LIST + do + cmd="$COMMAND_SUDO $FIND_CMD -L $directory/ -mindepth 1 -maxdepth 1 -type d >> $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + if [ $? != 0 ]; then + Logger "Could not enumerate directories in [$directory]." "ERROR" + if [ -f $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + if [ -f $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + retval=1 + else + retval=0 + fi + done + IFS=$OLD_IFS + return $retval +} + +function _ListRecursiveBackupDirectoriesRemote { + + OLD_IFS=$IFS + IFS=$PATH_SEPARATOR_CHAR + for directory in $RECURSIVE_DIRECTORY_LIST + do + cmd=$SSH_CMD' "'$COMMAND_SUDO' '$FIND_CMD' -L '$directory'/ -mindepth 1 -maxdepth 1 -type d" >> '$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + if [ $? != 0 ]; then + Logger "Could not enumerate directories in [$directory]." "ERROR" + if [ -f $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + if [ -f $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + retval=1 + else + retval=0 + fi + done + IFS=$OLD_IFS + return $retval +} + +function ListRecursiveBackupDirectories { + + Logger "Listing directories to backup." "NOTICE" + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]; then + _ListRecursiveBackupDirectoriesLocal + if [ $? != 0 ]; then + output_file="" + else + output_file="$RUN_DIR/$PROGRAM._ListRecursiveBackupDirectoriesLocal.$SCRIPT_PID" + fi + elif [ "$BACKUP_TYPE" == "pull" ]; then + _ListRecursiveBackupDirectoriesRemote + if [ $? != 0 ]; then + output_file="" + else + output_file="$RUN_DIR/$PROGRAM._ListRecursiveBackupDirectoriesRemote.$SCRIPT_PID" + fi + fi + + if [ -f "$output_file" ]; then + OLD_IFS=$IFS + IFS=$' \n' + for line in $(cat "$output_file") + do + file_exclude=0 + IFS=$PATH_SEPARATOR_CHAR + for k in $RECURSIVE_EXCLUDE_LIST + do + if [ "$k" == "$line" ]; then + file_exclude=1 + fi + done + IFS=$' \n' + + if [ $file_exclude -eq 0 ]; then + if [ "$FILE_RECURSIVE_BACKUP_TASKS" == "" ]; then + FILE_RECURSIVE_BACKUP_TASKS="$line" + FILE_SIZE_LIST="$(EscapeSpaces $line)" + else + FILE_RECURSIVE_BACKUP_TASKS="$FILE_RECURSIVE_BACKUP_TASKS$PATH_SEPARATOR_CHAR$line" + FILE_SIZE_LIST="$FILE_SIZE_LIST $(EscapeSpaces $line)" + fi + else + FILE_RECURSIVE_EXCLUDED_TASKS="$FILE_RECURSIVE_EXCLUDED_TASKS$PATH_SEPARATOR_CHAR$line" + fi + done + IFS=$OLD_IFS + fi + + OLD_IFS=$IFS + IFS=$PATH_SEPARATOR_CHAR + for directory in $DIRECTORY_LIST + do + FILE_SIZE_LIST="$FILE_SIZE_LIST $(EscapeSpaces $line)" + if [ "$FILE_BACKUP_TASKS" == "" ]; then + FILE_BACKUP_TASKS="$directory" + else + FILE_BACKUP_TASKS="$FILE_BACKUP_TASKS$PATH_SEPARATOR_CHAR$directory" + fi + done + IFS=$OLD_IFS +} + +function _GetDirectoriesSizeLocal { + local dir_list="${1}" + + cmd='echo "'$dir_list'" | xargs '$COMMAND_SUDO' du -cs | tail -n1 | cut -f1 > '$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + # $cmd will return 0 even if some errors found, so we need to check if there is an error output + if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID ]; then + Logger "Could not get files size for some or all directories." "ERROR" + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + else + Logger "File size fetched successfully." "NOTICE" + fi + + if [ -s "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + TOTAL_FILES_SIZE="$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" + else + TOTAL_FILES_SIZE=-1 + fi +} + +function _GetDirectoriesSizeRemote { + local dir_list="${1}" + + # Error output is different from stdout because not all files in list may fail at once + cmd=$SSH_CMD' "echo '$dir_list' | xargs '$COMMAND_SUDO' du -cs | tail -n1 | cut -f1" > '$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + # $cmd will return 0 even if some errors found, so we need to check if there is an error output + if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID ]; then + Logger "Could not get files size for some or all directories." "ERROR" + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + if [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + else + Logger "File size fetched successfully." "NOTICE" + fi + if [ -s "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then + TOTAL_FILES_SIZE="$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" + else + TOTAL_FILES_SIZE=-1 + fi +} + +function GetDirectoriesSize { + + Logger "Getting files size" "NOTICE" + + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]; then + if [ "$FILE_BACKUP" != "no" ]; then + _GetDirectoriesSizeLocal "$FILE_SIZE_LIST" + fi + elif [ "$BACKUP_TYPE" == "pull" ]; then + if [ "$FILE_BACKUP" != "no" ]; then + _GetDirectoriesSizeRemote "$FILE_SIZE_LIST" + fi + fi +} + +function _CreateStorageDirsLocal { + local dir_to_create="${1}" + + if [ ! -d "$dir_to_create" ]; then + $COMMAND_SUDO mkdir -p "$dir_to_create" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 + if [ $? != 0 ]; then + Logger "Cannot create directory [$dir_to_create]" "CRITICAL" + if [ -f $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID ]; then + Logger "Command output: $(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + fi + return 1 + fi + fi +} + +function _CreateStorageDirsRemote { + local dir_to_create="${1}" + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + cmd=$SSH_CMD' "if ! [ -d \"'$dir_to_create'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$dir_to_create'\"; fi" > '$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID' 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 1800 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot create remote directory [$dir_to_create]." "CRITICAL" + Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + return 1 + fi +} + +function CreateStorageDirectories { + + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "pull" ]; then + if [ "SQL_BACKUP" != "no" ]; then + _CreateStorageDirsLocal "$SQL_STORAGE" + if [ $? != 0 ]; then + CAN_BACKUP_SQL=0 + fi + fi + if [ "FILE_BACKUP" != "no" ]; then + _CreateStorageDirsLocal "$FILE_STORAGE" + if [ $? != 0 ]; then + CAN_BACKUP_FILES=0 + fi + fi + elif [ "$BACKUP_TYPE" == "push" ]; then + if [ "SQL_BACKUP" != "no" ]; then + _CreateStorageDirsRemote "$SQL_STORAGE" + if [ $? != 0 ]; then + CAN_BACKUP_SQL=0 + fi + fi + if [ "$FILE_BACKUP" != "no" ]; then + _CreateStorageDirsRemote "$FILE_STORAGE" + if [ $? != 0 ]; then + CAN_BACKUP_FILES=0 + fi + fi + fi +} + +function GetDiskSpaceLocal { + # GLOBAL VARIABLE DISK_SPACE to pass variable to parent function + # GLOBAL VARIABLE DRIVE to pass variable to parent function + local path_to_check="${1}" + + if [ -w "$path_to_check" ]; then + # Not elegant solution to make df silent on errors + $COMMAND_SUDO df -P "$path_to_check" > "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" 2>&1 + if [ $? != 0 ]; then + DISK_SPACE=0 + Logger "Cannot get disk space in [$path_to_check] on local system." "ERROR" + Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + else + DISK_SPACE=$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID | tail -1 | awk '{print $4}') + DRIVE=$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID | tail -1 | awk '{print $1}') + fi + else + Logger "Storage path [$path_to_check] does not exist or cannot write to it." "CRITICAL" + return 1 + fi +} + +function GetDiskSpaceRemote { + # USE GLOBAL VARIABLE DISK_SPACE to pass variable to parent function + local path_to_check="${1}" + + cmd=$SSH_CMD' "if [ -w \"'$path_to_check'\" ]; then '$COMMAND_SUDO' df -P \"'$path_to_check'\"; else exit 1; fi" > "'$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + if [ $? != 0 ]; then + DISK_SPACE=0 + Logger "Cannot get disk space in [$path_to_check] on remote system." "ERROR" + Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + return 1 + else + DISK_SPACE=$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID | tail -1 | awk '{print $4}') + DRIVE=$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID | tail -1 | awk '{print $1}') + fi +} + +function CheckDiskSpace { + # USE OF GLOBAL VARIABLES TOTAL_DATABASES_SIZE, TOTAL_FILES_SIZE, BACKUP_SIZE_MINIMUM, STORAGE_WARN_SIZE, STORAGE_SPACE + + + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "pull" ]; then + if [ "$SQL_BACKUP" != "no" ]; then + GetDiskSpaceLocal "$SQL_STORAGE" + if [ $? != 0 ]; then + SQL_DISK_SPACE=0 + CAN_BACKUP_SQL=0 + else + SQL_DISK_SPACE=$DISK_SPACE + SQL_DRIVE=$DRIVE + fi + fi + if [ "$FILE_BACKUP" != "no" ]; then + GetDiskSpaceLocal "$FILE_STORAGE" + if [ $? != 0 ]; then + FILE_DISK_SPACE=0 + CAN_BACKUP_FILES=0 + else + FILE_DISK_SPACE=$DISK_SPACE + FILE_DRIVE=$DRIVE + fi + fi + elif [ "$BACKUP_TYPE" == "push" ]; then + if [ "$SQL_BACKUP" != "no" ]; then + GetDiskSpaceRemote "$SQL_STORAGE" + if [ $? != 0 ]; then + SQL_DISK_SPACE=0 + else + SQL_DISK_SPACE=$DISK_SPACE + SQL_DRIVE=$DRIVE + fi + fi + if [ "$FILE_BACKUP" != "no" ]; then + GetDiskSpaceRemote "$FILE_STORAGE" + if [ $? != 0 ]; then + FILE_DISK_SPACE=0 + else + FILE_DISK_SPACE=$DISK_SPACE + FILE_DRIVE=$DRIVE + fi + fi + fi + + if [ "$TOTAL_DATABASES_SIZE" == "" ]; then + TOTAL_DATABASES_SIZE=-1 + fi + if [ "$TOTAL_FILES_SIZE" == "" ]; then + TOTAL_FILES_SIZE=-1 + fi + + if [ "$SQL_BACKUP" != "no" ] && [ $CAN_BACKUP_SQL -eq 1 ]; then + if [ $SQL_DISK_SPACE -eq 0 ]; then + Logger "Storage space in [$SQL_STORAGE] reported to be 0Ko." "WARN" + fi + if [ $SQL_DISK_SPACE -lt $TOTAL_DATABASES_SIZE ]; then + Logger "Disk space in [$SQL_STORAGE] may be insufficient to backup SQL ($SQL_DISK_SPACE Ko available in $SQL_DRIVE) (non compressed databases calculation)." "WARN" + fi + if [ $SQL_DISK_SPACE -lt $SQL_WARN_MIN_SPACE ]; then + Logger "Disk space in [$SQL_STORAGE] is lower than warning value [$SQL_WARN_MIN_SPACE Ko]." "WARN" + fi + Logger "SQL storage Space: $SQL_DISK_SPACE Ko - Databases size: $TOTAL_DATABASES_SIZE Ko" "NOTICE" + fi + + if [ "$FILE_BACKUP" != "no" ] && [ $CAN_BACKUP_FILES -eq 1 ]; then + if [ $FILE_DISK_SPACE -eq 0 ]; then + Logger "Storage space in [$FILE_STORAGE] reported to be 0 Ko." "WARN" + fi + if [ $FILE_DISK_SPACE -lt $TOTAL_FILES_SIZE ]; then + Logger "Disk space in [$FILE_STORAGE] may be insufficient to backup files ($FILE_DISK_SPACE Ko available in $FILE_DRIVE)." "WARN" + fi + if [ $FILE_DISK_SPACE -lt $FILE_WARN_MIN_SPACE ]; then + Logger "Disk space in [$FILE_STORAGE] is lower than warning value [$FILE_WARN_MIN_SPACE Ko]." "WARN" + fi + Logger "File storage space: $FILE_DISK_SPACE Ko - Files size: $TOTAL_FILES_SIZE Ko" "NOTICE" + fi + + if [ $BACKUP_SIZE_MINIMUM -gt $(($TOTAL_DATABASES_SIZE+$TOTAL_FILES_SIZE)) ] && [ "$GET_BACKUP_SIZE" != "no" ]; then + Logger "Backup size is smaller than expected." "WARN" + fi +} + +function _BackupDatabaseLocalToLocal { + local database="${1}" # Database to backup + local export_options="${2}" # export options + + + local dry_sql_cmd="mysqldump -u $SQL_USER $export_options --database $database $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS > /dev/null 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" + local sql_cmd="mysqldump -u $SQL_USER $export_options --database $database $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS > $SQL_STORAGE/$database.sql$COMPRESSION_EXTENSION 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" + + if [ $_DRYRUN -ne 1 ]; then + Logger "cmd: $sql_cmd" "DEBUG" + eval "$sql_cmd" & + else + Logger "cmd: $dry_sql_cmd" "DEBUG" + eval "$dry_sql_cmd" & + fi + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + local retval=$? + if [ -s "$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + return $retval +} + +function _BackupDatabaseLocalToRemote { + local database="${1}" # Database to backup + local export_options="${2}" # export options + + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + + #TODO-v2.0: cannot catch mysqldump warnings + local dry_sql_cmd="mysqldump -u $SQL_USER $export_options --database $database $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS > /dev/null 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" + local sql_cmd="mysqldump -u $SQL_USER $export_options --database $database $COMPRESSION_PROGRAM $COMPRESSION_OPTIONS | $SSH_CMD '$COMMAND_SUDO tee \"$SQL_STORAGE/$database.sql$COMPRESSION_EXTENSION\" > /dev/null' 2> $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" + + if [ $_DRYRUN -ne 1 ]; then + Logger "cmd: $sql_cmd" "DEBUG" + eval "$sql_cmd" & + else + Logger "cmd: $dry_sql_cmd" "DEBUG" + eval "$dry_sql_cmd" & + fi + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + local retval=$? + if [ -s "$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + return $retval +} + +function _BackupDatabaseRemoteToLocal { + local database="${1}" # Database to backup + local export_options="${2}" # export options + + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + + local dry_sql_cmd=$SSH_CMD' "mysqldump -u '$SQL_USER' '$export_options' --database '$database' '$COMPRESSION_PROGRAM' '$COMPRESSION_OPTIONS'" > /dev/null 2> "'$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID'"' + local sql_cmd=$SSH_CMD' "mysqldump -u '$SQL_USER' '$export_options' --database '$database' '$COMPRESSION_PROGRAM' '$COMPRESSION_OPTIONS'" > "'$SQL_STORAGE/$database.sql$COMPRESSION_EXTENSION'" 2> "'$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID'"' + + if [ $_DRYRUN -ne 1 ]; then + Logger "cmd: $sql_cmd" "DEBUG" + eval "$sql_cmd" & + else + Logger "cmd: $dry_sql_cmd" "DEBUG" + eval "$dry_sql_cmd" & + fi + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + local retval=$? + if [ -s "$RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID" ]; then + Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.error.$SCRIPT_PID)" "ERROR" + fi + return $retval +} + +function BackupDatabase { + local database="${1}" + + # Hack to prevent warning on table mysql.events, some mysql versions don't support --skip-events, prefer using --ignore-table + if [ "$database" == "mysql" ]; then + local mysql_options='--skip-lock-tables --single-transaction --ignore-table=mysql.event' + else + local mysql_options='--skip-lock-tables --single-transaction' + fi + + if [ "$BACKUP_TYPE" == "local" ]; then + _BackupDatabaseLocalToLocal "$database" "$mysql_options" + elif [ "$BACKUP_TYPE" == "pull" ]; then + _BackupDatabaseRemoteToLocal "$database" "$mysql_options" + elif [ "$BACKUP_TYPE" == "push" ]; then + _BackupDatabaseLocalToRemote "$database" "$mysql_options" + fi + + if [ $? -ne 0 ]; then + Logger "Backup failed." "ERROR" + else + Logger "Backup succeeded." "NOTICE" + fi +} + +function BackupDatabases { + + local database + + OLD_IFS=$IFS + IFS=$' \t\n' + for database in $SQL_BACKUP_TASKS + do + Logger "Backing up database [$database]." "NOTICE" + BackupDatabase $database & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $FUNCNAME + CheckTotalExecutionTime + done + IFS=$OLD_IFS +} + +function Rsync { + local backup_directory="${1}" # Which directory to backup + local is_recursive="${2}" # Backup only files at toplevel of directory + + + if [ "$KEEP_ABSOLUTE_PATHS" == "yes" ]; then + local file_storage_path="$(dirname $FILE_STORAGE$backup_directory)" + else + local file_storage_path="$FILE_STORAGE" + fi + + ## Manage to backup recursive directories lists files only (not recursing into subdirectories) + if [ "$is_recursive" == "no-recurse" ]; then + # Fixes symlinks to directories in target cannot be deleted when backing up root directory without recursion, and excludes subdirectories + RSYNC_NO_RECURSE_ARGS=" -k --exclude=*/*/" + else + RSYNC_NO_RECURSE_ARGS="" + fi + + # Creating subdirectories because rsync cannot handle mkdir -p + if [ ! -d "$file_storage_path/$backup_directory" ]; then + $COMMAND_SUDO mkdir -p "$file_storage_path/$backup_directory" + if [ $? != 0 ]; then + Logger "Cannot create storage path [$file_storage_path/$backup_directory]." "ERROR" + fi + fi + + if [ "$BACKUP_TYPE" == "local" ]; then + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + elif [ "$BACKUP_TYPE" == "pull" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + elif [ "$BACKUP_TYPE" == "push" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$backup_directory\" \"$REMOTE_USER@$REMOTE_HOST:$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + fi + + Logger "cmd: $rsync_cmd" "DEBUG" + eval "$rsync_cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + if [ $? != 0 ]; then + Logger "Failed to backup [$backup_directory] to [$file_storage_path]." "ERROR" + Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + else + Logger "File backup succeed." "NOTICE" + fi +} + +function Duplicity { + local backup_directory="${1}" # Which directory to backup + local is_recursive="${2}" # Backup only files at toplevel of directory + + + Logger "Encrpytion not supported yet ! No backup done." "CRITICAL" + return 1 + + if [ "$KEEP_ABSOLUTE_PATHS" == "yes" ]; then + local file_storage_path="$(dirname $FILE_STORAGE$backup_directory)" + else + local file_storage_path="$FILE_STORAGE" + fi + + if [ "$BACKUP_TYPE" == "local" ]; then + duplicity_cmd="" + elif [ "$BACKUP_TYPE" == "pull" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + duplicity_cmd="" + elif [ "$BACKUP_TYPE" == "push" ]; then + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + duplicity_cmd="" + fi + + Logger "cmd: $duplicity_cmd" "DEBUG" + eval "$duplicity_cmd" & + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $FUNCNAME + if [ $? != 0 ]; then + Logger "Failed to backup [$backup_directory] to [$file_storage_path]." "ERROR" + Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + else + Logger "File backup succeed." "NOTICE" + fi + +} + +function FilesBackup { + + OLD_IFS=$IFS + IFS=$PATH_SEPARATOR_CHAR + # Backup non recursive directories + for BACKUP_TASK in $FILE_BACKUP_TASKS + do + Logger "Beginning file backup of [$BACKUP_TASK]." "NOTICE" + if [ "$ENCRYPTION" == "yes" ]; then + Duplicity "$BACKUP_TASK" "recurse" + else + Rsync "$BACKUP_TASK" "recurse" + fi + CheckTotalExecutionTime + done + + ## Backup files at root of DIRECTORIES_RECURSE_LIST directories + for BACKUP_TASK in $RECURSIVE_DIRECTORY_LIST + do + Logger "Beginning non recursive file backup of [$BACKUP_TASK]." "NOTICE" + if [ "$ENCRYPTION" == "yes" ]; then + Duplicity "$BACKUP_TASK" "no-recurse" + else + Rsync "$BACKUP_TASK" "no-recurse" + fi + CheckTotalExecutionTime + done + + # Backup sub directories of recursive directories + for BACKUP_TASK in $FILE_RECURSIVE_BACKUP_TASKS + do + Logger "Beginning recursive file backup of [$BACKUP_TASK]." "NOTICE" + if [ "$ENCRYPTION" == "yes" ]; then + Duplicity "$BACKUP_TASK" "recurse" + else + Rsync "$BACKUP_TASK" "recurse" + fi + CheckTotalExecutionTime + done + IFS=$OLD_IFS +} + +function CheckTotalExecutionTime { + + #### Check if max execution time of whole script as been reached + if [ $SECONDS -gt $SOFT_MAX_EXEC_TIME_TOTAL ]; then + Logger "Max soft execution time of the whole backup exceeded while backing up [$BACKUP_TASK]." "ERROR" + WARN_ALERT=1 + SendAlert + if [ $SECONDS -gt $HARD_MAX_EXEC_TIME_TOTAL ] && [ $HARD_MAX_EXEC_TIME_TOTAL -ne 0 ]; then + Logger "Max hard execution time of the whole backup exceeded while backing up [$BACKUP_TASK], stopping backup process." "CRITICAL" + exit 1 + fi + fi +} + +function RsyncExcludePattern { + + # Disable globbing so wildcards from exclusions do not get expanded + set -f + rest="$RSYNC_EXCLUDE_PATTERN" + while [ -n "$rest" ] + do + # Take the string until first occurence until $PATH_SEPARATOR_CHAR + str=${rest%%;*} + # Handle the last case + if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ]; then + rest= + else + # Cut everything before the first occurence of $PATH_SEPARATOR_CHAR + rest=${rest#*$PATH_SEPARATOR_CHAR} + fi + + if [ "$RSYNC_EXCLUDE" == "" ]; then + RSYNC_EXCLUDE="--exclude=\"$str\"" + else + RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$str\"" + fi + done + set +f +} + +function RsyncExcludeFrom { + + if [ ! $RSYNC_EXCLUDE_FROM == "" ]; then + ## Check if the exclude list has a full path, and if not, add the config file path if there is one + if [ "$(basename $RSYNC_EXCLUDE_FROM)" == "$RSYNC_EXCLUDE_FROM" ]; then + RSYNC_EXCLUDE_FROM=$(dirname $ConfigFile)/$RSYNC_EXCLUDE_FROM + fi + + if [ -e $RSYNC_EXCLUDE_FROM ]; then + RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\"" + fi + fi +} + +function _RotateBackupsLocal { + local backup_path="${1}" + + OLD_IFS=$IFS + IFS=$'\t\n' + for backup in $(ls -I "*.$PROGRAM.*" "$backup_path") + do + copy=$ROTATE_COPIES + while [ $copy -gt 1 ] + do + if [ $copy -eq $ROTATE_COPIES ]; then + cmd="$COMMAND_SUDO rm -rf \"$backup_path/$backup.$PROGRAM.$copy\"" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot delete oldest copy [$backup_path/$backup.$PROGRAM.$copy]." "ERROR" + fi + fi + path="$backup_path/$backup.$PROGRAM.$(($copy-1))" + if [[ -f $path || -d $path ]]; then + cmd="$COMMAND_SUDO mv \"$path\" \"$backup_path/$backup.$PROGRAM.$copy\"" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot move [$path] to [$backup_path/$backup.$PROGRAM.$copy]." "ERROR" + fi + + fi + copy=$(($copy-1)) + done + + # Latest file backup will not be moved if script configured for remote backup so next rsync execution will only do delta copy instead of full one + if [[ $backup == *.sql.* ]]; then + cmd="$COMMAND_SUDO mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + + elif [ "$REMOTE_OPERATION" == "yes" ]; then + cmd="$COMMAND_SUDO cp -R \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot copy [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + + else + cmd="$COMMAND_SUDO mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + WaitForTaskCompletion $! 720 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + fi + done + IFS=$OLD_IFS +} + +function _RotateBackupsRemote { + local backup_path="${1}" +$SSH_CMD PROGRAM=$PROGRAM REMOTE_OPERATION=$REMOTE_OPERATION _DEBUG=$_DEBUG COMMAND_SUDO=$COMMAND_SUDO ROTATE_COPIES=$ROTATE_COPIES backup_path="$backup_path" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" 2>&1 & + +function _RemoteLogger { + local value="${1}" # What to log + echo -e "$value" +} + +function RemoteLogger { + local value="${1}" # Sentence to log (in double quotes) + local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL + + prefix="REMOTE TIME: $SECONDS - " + + if [ "$level" == "CRITICAL" ]; then + _RemoteLogger "$prefix\e[41m$value\e[0m" + return + elif [ "$level" == "ERROR" ]; then + _RemoteLogger "$prefix\e[91m$value\e[0m" + return + elif [ "$level" == "WARN" ]; then + _RemoteLogger "$prefix\e[93m$value\e[0m" + return + elif [ "$level" == "NOTICE" ]; then + _RemoteLogger "$prefix$value" + return + elif [ "$level" == "DEBUG" ]; then + if [ "$_DEBUG" == "yes" ]; then + _RemoteLogger "$prefix$value" + return + fi + else + _RemoteLogger "\e[41mLogger function called without proper loglevel.\e[0m" + _RemoteLogger "$prefix$value" + fi +} + +function _RotateBackupsRemoteSSH { + OLD_IFS=$IFS + IFS=$' \t\n' + for backup in $(ls -I "*.$PROGRAM.*" "$backup_path") + do + copy=$ROTATE_COPIES + while [ $copy -gt 1 ] + do + if [ $copy -eq $ROTATE_COPIES ]; then + cmd="$COMMAND_SUDO rm -rf \"$backup_path/$backup.$PROGRAM.$copy\"" + RemoteLogger "cmd: $cmd" "DEBUG" + eval "$cmd" + if [ $? != 0 ]; then + RemoteLogger "Cannot delete oldest copy [$backup_path/$backup.$PROGRAM.$copy]." "ERROR" + fi + fi + path="$backup_path/$backup.$PROGRAM.$(($copy-1))" + if [[ -f $path || -d $path ]]; then + cmd="$COMMAND_SUDO mv \"$path\" \"$backup_path/$backup.$PROGRAM.$copy\"" + RemoteLogger "cmd: $cmd" "DEBUG" + eval "$cmd" + if [ $? != 0 ]; then + RemoteLogger "Cannot move [$path] to [$backup_path/$backup.$PROGRAM.$copy]." "ERROR" + fi + + fi + copy=$(($copy-1)) + done + + # Latest file backup will not be moved if script configured for remote backup so next rsync execution will only do delta copy instead of full one + if [[ $backup == *.sql.* ]]; then + cmd="$COMMAND_SUDO mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + RemoteLogger "cmd: $cmd" "DEBUG" + eval "$cmd" + if [ $? != 0 ]; then + RemoteLogger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + + elif [ "$REMOTE_OPERATION" == "yes" ]; then + cmd="$COMMAND_SUDO cp -R \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + RemoteLogger "cmd: $cmd" "DEBUG" + eval "$cmd" + if [ $? != 0 ]; then + RemoteLogger "Cannot copy [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + + else + cmd="$COMMAND_SUDO mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\"" + RemoteLogger "cmd: $cmd" "DEBUG" + eval "$cmd" + if [ $? != 0 ]; then + RemoteLogger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR" + fi + fi + done + IFS=$OLD_IFS +} + + _RotateBackupsRemoteSSH + +ENDSSH + + WaitForTaskCompletion $! 1800 0 $FUNCNAME + if [ $? != 0 ]; then + Logger "Could not rotate backups in [$backup_path]." "ERROR" + Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "ERROR" + else + Logger "Remote rotation succeed." "NOTICE" + fi ## Need to add a trivial sleep time to give ssh time to log to local file + #sleep 5 + + +} + +function RotateBackups { + local backup_path="${1}" + + Logger "Rotating backups." "NOTICE" + + if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "pull" ]; then + _RotateBackupsLocal "$backup_path" + elif [ "$BACKUP_TYPE" == "push" ]; then + _RotateBackupsRemote "$backup_path" + fi +} + +function Init { + + trap TrapStop SIGINT SIGQUIT SIGKILL SIGTERM SIGHUP + trap TrapQuit EXIT + + ## Test if target dir is a ssh uri, and if yes, break it down it its values + if [ "${REMOTE_SYSTEM_URI:0:6}" == "ssh://" ] && [ "$BACKUP_TYPE" != "local" ]; then + REMOTE_OPERATION="yes" + + # remove leadng 'ssh://' + uri=${REMOTE_SYSTEM_URI#ssh://*} + if [[ "$uri" == *"@"* ]]; then + # remove everything after '@' + REMOTE_USER=${uri%@*} + else + REMOTE_USER=$LOCAL_USER + fi + + if [ "$SSH_RSA_PRIVATE_KEY" == "" ]; then + SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa + fi + + # remove everything before '@' + _hosturiandpath=${uri#*@} + # remove everything after first '/' + _hosturi=${_hosturiandpath%%/*} + if [[ "$_hosturi" == *":"* ]]; then + REMOTE_PORT=${_hosturi##*:} + else + REMOTE_PORT=22 + fi + REMOTE_HOST=${_hosturi%%:*} + fi + + ## Add update to default RSYNC_ARGS + RSYNC_ARGS=$RSYNC_ARGS"u" + + if [ $_VERBOSE -eq 1 ]; then + RSYNC_ARGS=$RSYNC_ARGS"i" + fi + + if [ "$PARTIAL" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" + RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$PARTIAL_DIR\"" + fi + + if [ "$DELETE_VANISHED_FILES" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --delete" + fi + + if [ "$DELTA_COPIES" != "no" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file" + else + RSYNC_ARGS=$RSYNC_ARGS" --whole-file" + fi + + if [ $stats -eq 1 ]; then + RSYNC_ARGS=$RSYNC_ARGS" --stats" + fi + + ## Fix for symlink to directories on target cannot get updated + RSYNC_ARGS=$RSYNC_ARGS" --force" +} + +function Main { + + if [ "$SQL_BACKUP" != "no" ] && [ $CAN_BACKUP_SQL -eq 1 ]; then + ListDatabases + fi + if [ "$FILE_BACKUP" != "no" ] && [ $CAN_BACKUP_FILES -eq 1 ]; then + ListRecursiveBackupDirectories + if [ "$GET_BACKUP_SIZE" != "no" ]; then + GetDirectoriesSize + else + TOTAL_FILE_SIZE=0 + fi + fi + + if [ "$CREATE_DIRS" != "no" ]; then + CreateStorageDirectories + fi + CheckDiskSpace # Actual backup process - if [ "$BACKUP_SQL" != "no" ] - then - if [ $dryrun -ne 1 ] - then - if [ "$ROTATE_BACKUPS" == "yes" ] - then - RotateBackups $LOCAL_SQL_STORAGE - fi + if [ "$SQL_BACKUP" != "no" ] && [ $CAN_BACKUP_SQL -eq 1 ]; then + if [ $_DRYRUN -ne 1 ] && [ "$ROTATE_SQL_BACKUPS" == "yes" ]; then + RotateBackups "$SQL_STORAGE" fi BackupDatabases fi - if [ "$BACKUP_FILES" != "no" ] - then - if [ $dryrun -ne 1 ] - then - if [ "$ROTATE_BACKUPS" == "yes" ] - then - RotateBackups $LOCAL_FILE_STORAGE - fi + if [ "$FILE_BACKUP" != "no" ] && [ $CAN_BACKUP_FILES -eq 1 ]; then + if [ $_DRYRUN -ne 1 ] && [ "$ROTATE_FILE_BACKUPS" == "yes" ]; then + RotateBackups "$FILE_STORAGE" fi ## Add Rsync exclude patterns - RsyncExcludePattern - ## Add Rsync exclude from file - RsyncExcludeFrom - + RsyncExcludePattern + ## Add Rsync exclude from file + RsyncExcludeFrom FilesBackup fi - # Be a happy sysadmin (and drink a coffee ? Nahh... it's past midnight.) } -function Usage -{ - echo "$PROGRAM $PROGRAM_VERSION $PROGRAM_BUILD" +function Usage { + + + if [ "$IS_STABLE" != "yes" ]; then + echo -e "\e[93mThis is an unstable dev build. Please use with caution.\e[0m" + fi + + echo "$PROGRAM $PROGRAM_VERSION $PROGRAM_BUILD" echo "$AUTHOR" echo "$CONTACT" echo "" - echo "usage: obackup /path/to/backup.conf [--dry] [--silent] [--verbose] [--no-maxtime]" + echo "usage: obackup.sh /path/to/backup.conf [OPTIONS]" echo "" + echo "OPTIONS:" echo "--dry: will run obackup without actually doing anything, just testing" echo "--silent: will run obackup without any output to stdout, usefull for cron backups" echo "--verbose: adds command outputs" - echo "--stats Adds rsync transfer statistics to verbose output" - echo "--partial Allows rsync to keep partial downloads that can be resumed later (experimental)" + echo "--stats Adds rsync transfer statistics to verbose output" + echo "--partial Allows rsync to keep partial downloads that can be resumed later (experimental)" echo "--no-maxtime disables any soft and hard execution time checks" echo "--delete Deletes files on destination that vanished on source" - echo "--dontgetsize Does not try to evaluate backup size" + echo "--dontgetsize Does not try to evaluate backup size" exit 128 } # Command line argument flags -dryrun=0 -silent=0 +_DRYRUN=0 +_SILENT=0 no_maxtime=0 -if [ "$DEBUG" == "yes" ] -then - verbose=1 -else - verbose=0 -fi - dontgetsize=0 +dontgetsize=0 stats=0 PARTIAL=0 -# Alert flags -soft_alert_total=0 -error_alert=0 -function GetCommandlineArguments -{ - if [ $# -eq 0 ] - then +function GetCommandlineArguments { + if [ $# -eq 0 ]; then Usage fi @@ -1527,13 +2019,13 @@ function GetCommandlineArguments do case $i in --dry) - dryrun=1 + _DRYRUN=1 ;; --silent) - silent=1 + _SILENT=1 ;; --verbose) - verbose=1 + _VERBOSE=1 ;; --stats) stats=1 @@ -1548,7 +2040,7 @@ function GetCommandlineArguments DELETE_VANISHED_FILES="yes" ;; --dontgetsize) - DISABLE_GET_BACKUP_FILE_SIZE="yes" + GET_BACKUP_SIZE="no" ;; --help|-h|--version|-v) Usage @@ -1559,61 +2051,46 @@ function GetCommandlineArguments GetCommandlineArguments "$@" CheckEnvironment -if [ $? == 0 ] -then - if [ "$1" != "" ] - then - LoadConfigFile $1 - if [ $? == 0 ] - then - if [ "$LOGFILE" == "" ] - then - if [ -w /var/log ] - then - LOG_FILE=/var/log/obackup_$BACKUP_ID.log - else - LOG_FILE=./obackup_$BACKUP_ID.log - fi - else - LOG_FILE="$LOGFILE" - fi - - GetLocalOS - InitLocalOSSettings - - Init - GetRemoteOS - InitRemoteOSSettings - DATE=$(date) - Log "Obackup starting" "noprefix" - Log "--------------------------------------------------------------------" - Log "$DRY_WARNING $DATE - $PROGRAM v$PROGRAM_VERSION script begin." - Log "--------------------------------------------------------------------" - Log "Backup task [$BACKUP_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" - - if [ $no_maxtime -eq 1 ] - then - SOFT_MAX_EXEC_TIME_DB_TASK=0 - SOFT_MAX_EXEC_TIME_FILE_TASK=0 - HARD_MAX_EXEC_TIME_DB_TASK=0 - HARD_MAX_EXEC_TIME_FILE_TASK=0 - HARD_MAX_EXEC_TIME_TOTAL=0 - fi - OLD_IFS=$IFS - RunBeforeHook - Main - IFS=$OLD_IFS - RunAfterHook - CleanUp - else - LogError "Configuration file could not be loaded." - exit 1 - fi +LoadConfigFile "$1" +if [ "$LOGFILE" == "" ]; then + if [ -w /var/log ]; then + LOG_FILE=/var/log/$PROGRAM.$INSTANCE_ID.log else - LogError "No configuration file provided." - exit 1 + LOG_FILE=./$PROGRAM.$INSTANCE_ID.log fi else - LogError "Environment not suitable to run obackup." - exit 1 + LOG_FILE="$LOGFILE" fi + +if [ "$IS_STABLE" != "yes" ]; then + Logger "This is an unstable dev build. Please use with caution." "WARN" +fi + + +GetLocalOS +InitLocalOSSettings +PreInit +Init +PostInit +CheckCurrentConfig +if [ "$REMOTE_OPERATION" == "yes" ]; then + GetRemoteOS + InitRemoteOSSettings +fi +DATE=$(date) +Logger "--------------------------------------------------------------------" "NOTICE" +Logger "$DRY_WARNING $DATE - $PROGRAM v$PROGRAM_VERSION $BACKUP_TYPE script begin." "NOTICE" +Logger "--------------------------------------------------------------------" "NOTICE" +Logger "Backup instance [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE" + +if [ $no_maxtime -eq 1 ]; then + SOFT_MAX_EXEC_TIME_DB_TASK=0 + SOFT_MAX_EXEC_TIME_FILE_TASK=0 + HARD_MAX_EXEC_TIME_DB_TASK=0 + HARD_MAX_EXEC_TIME_FILE_TASK=0 + HARD_MAX_EXEC_TIME_TOTAL=0 +fi + +RunBeforeHook +Main +RunAfterHook diff --git a/ssh_filter.sh b/ssh_filter.sh index 416a814..b1c2754 100755 --- a/ssh_filter.sh +++ b/ssh_filter.sh @@ -1,6 +1,6 @@ -#!/bin/bash +#!/usr/bin/env bash -##### Osync ssh command filter build 2015070203 +##### osync / obackup ssh command filter build 2015102701 ##### This script should be located in /usr/local/bin in the remote system to sync / backup ##### It will filter the commands that can be run remotely via ssh. ##### Please chmod 755 and chown root:root this file @@ -19,77 +19,77 @@ CMD3= LOG_FILE=~/.ssh/ssh_filter.log -function Log -{ +function Log { DATE=$(date) echo "$DATE - $1" >> $LOG_FILE } -function Go -{ +function Go { eval $SSH_ORIGINAL_COMMAND } case ${SSH_ORIGINAL_COMMAND%% *} in "$RSYNC_EXECUTABLE") Go ;; + "echo") + Go ;; "find") Go ;; "du") Go ;; - "mysql") + "mkdir") Go ;; - "mysqldump") + "rm") + Go ;; + "df") + Go ;; + "mv") Go ;; "$CMD1") - if [ "$CMD1" != "" ] - then + if [ "$CMD1" != "" ]; then Go ;; fi "$CMD2") - if [ "$CMD2" != "" ] - then + if [ "$CMD2" != "" ]; then Go ;; fi "$CMD3") - if [ "$CMD3" != "" ] - then + if [ "$CMD3" != "" ]; then Go ;; fi "sudo") - if [ "$SUDO_EXEC" == "yes" ] - then - if [[ "$SSH_ORIGINAL_COMMAND" == "sudo $RSYNC_EXECUTABLE"* ]] + if [ "$SUDO_EXEC" == "yes" ]; then + if [[ "$SSH_ORIGINAL_COMMAND" == "sudo $RSYNC_EXECUTABLE"* ]]; then + Go + elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo du"* ]]; then + Go + elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo find"* ]]; then + Go + elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo mkdir"* ]] then Go - elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo du"* ]] + elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo rm"* ]] then Go - elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo find"* ]] + elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo echo"* ]] then Go - elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo mysql"* ]] - then - Go - elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo mysqldump"* ]] - then - Go - elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo $CMD1"* ]] + elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo df"* ]] then - if [ "$CMD1" != "" ] - then + Go + elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo mv"* ]] + then + Go + elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo $CMD1"* ]]; then + if [ "$CMD1" != "" ]; then Go fi - elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo $CMD2"* ]] - then - if [ "$CMD2" != "" ] - then + elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo $CMD2"* ]]; then + if [ "$CMD2" != "" ]; then Go fi - elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo $CMD3"* ]] - then - if [ "$CMD3" != "" ] - then + elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo $CMD3"* ]]; then + if [ "$CMD3" != "" ]; then Go fi else