diff --git a/obackup.sh b/obackup.sh deleted file mode 100755 index d0b8799..0000000 --- a/obackup.sh +++ /dev/null @@ -1,1085 +0,0 @@ -#!/bin/bash - -###### Remote (or local) backup script for files & databases -###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr) -OBACKUP_VERSION=1.84RC1 -OBACKUP_BUILD=1607201301 - -DEBUG=no -SCRIPT_PID=$$ - -LOCAL_USER=$(whoami) -LOCAL_HOST=$(hostname) - -## Default log file until config file is loaded -LOG_FILE=/var/log/obackup.log - -## Log a state message every $KEEP_LOGGING seconds. Should generally not be equal to soft or hard execution time so your log won't be unnecessary big. -KEEP_LOGGING=1801 - -## Global variables and forked command results -DATABASES_TO_BACKUP="" # Processed list of DBs that will be backed up -DATABASES_EXCLUDED_LIST="" # Processed list of DBs that won't be backed up -TOTAL_DATABASES_SIZE=0 # Total DB size of $DATABASES_TO_BACKUP -DIRECTORIES_RECURSE_TO_BACKUP="" # Processed list of recursive directories that will be backed up -DIRECTORIES_EXCLUDED_LIST="" # Processed list of recursive directorires that won't be backed up -DIRECTORIES_TO_BACKUP="" # Processed list of all directories to backup -TOTAL_FILES_SIZE=0 # Total file size of $DIRECTORIES_TO_BACKUP - -# /dev/shm/obackup_dblist_$SCRIPT_PID Databases list and sizes -# /dev/shm/obackup_dirs_recurse_list_$SCRIPT_PID Recursive directories list -# /dev/shm/obackup_local_sql_storage_$SCRIPT_PID Local free space for sql backup -# /dev/shm/obackup_local_file_storage_$SCRIPT_PID Local free space for file backup -# /dev/shm/obackup_fsize_$SCRIPT_PID Size of $DIRECTORIES_TO_BACKUP -# /dev/shm/obackup_rsync_output_$SCRIPT_PID Output of Rsync command -# /dev/shm/obackup_config_$SCRIPT_PID Parsed configuration file -# /dev/shm/obackup_run_local_$SCRIPT_PID Output of command to be run localy -# /dev/shm/obackup_run_remote_$SCRIPT_PID Output of command to be run remotely - -function Log -{ - echo "TIME: $SECONDS - $1" >> "$LOG_FILE" - if [ $silent -eq 0 ] - then - echo "TIME: $SECONDS - $1" - fi -} - -function LogError -{ - Log "$1" - error_alert=1 -} - -function TrapError -{ - local JOB="$0" - local LINE="$1" - local CODE="${2:-1}" - if [ $silent -eq 0 ] - then - echo " /!\ Error in ${JOB}: Near line ${LINE}, exit code ${CODE}" - fi -} - -function TrapStop -{ - LogError " /!\ WARNING: Manual exit of backup script. Backups may be in inconsistent state." - if [ "$DEBUG" == "no" ] - then - CleanUp - fi - exit 1 -} - -function Spinner -{ - if [ $silent -eq 1 ] - then - return 1 - fi - - case $toggle - in - 1) - echo -n $1" \ " - echo -ne "\r" - toggle="2" - ;; - - 2) - echo -n $1" | " - echo -ne "\r" - toggle="3" - ;; - - 3) - echo -n $1" / " - echo -ne "\r" - toggle="4" - ;; - - *) - echo -n $1" - " - echo -ne "\r" - toggle="1" - ;; - esac -} - -function Dummy -{ - exit 1; -} - -function StripQuotes -{ - echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") -} - -function EscapeSpaces -{ - echo $(echo $1 | sed 's/ /\\ /g') -} - -function CleanUp -{ - rm -f /dev/shm/obackup_dblist_$SCRIPT_PID - rm -f /dev/shm/obackup_local_sql_storage_$SCRIPT_PID - rm -f /dev/shm/obackup_local_file_storage_$SCRIPT_PID - rm -f /dev/shm/obackup_dirs_recurse_list_$SCRIPT_PID - rm -f /dev/shm/obackup_fsize_$SCRIPT_PID - rm -f /dev/shm/obackup_rsync_output_$SCRIPT_PID - rm -f /dev/shm/obackup_config_$SCRIPT_PID - rm -f /dev/shm/obackup_run_local_$SCRIPT_PID - rm -f /dev/shm/obackup_run_remote_$SCRIPT_PID -} - -function SendAlert -{ - CheckConnectivityRemoteHost - CheckConnectivity3rdPartyHosts - cat "$LOG_FILE" | gzip -9 > /tmp/obackup_lastlog.gz - if type -p mutt > /dev/null 2>&1 - then - echo $MAIL_ALERT_MSG | $(which mutt) -x -s "Backup alert for $BACKUP_ID" $DESTINATION_MAILS -a /tmp/obackup_lastlog.gz - if [ $? != 0 ] - then - Log "WARNING: Cannot send alert email via $(which mutt) !!!" - else - Log "Sent alert mail using mutt." - fi - elif type -p mail > /dev/null 2>&1 - then - echo $MAIL_ALERT_MSG | $(which mail) -a /tmp/obackup_lastlog.gz -s "Backup alert for $BACKUP_ID" $DESTINATION_MAILS - if [ $? != 0 ] - then - Log "WARNING: Cannot send alert email via $(which mail) with attachments !!!" - echo $MAIL_ALERT_MSG | $(which mail) -s "Backup alert for $BACKUP_ID" $DESTINATION_MAILS - if [ $? != 0 ] - then - Log "WARNING: Cannot send alert email via $(which mail) without attachments !!!" - else - Log "Sent alert mail using mail command without attachment." - fi - else - Log "Sent alert mail using mail command." - fi - else - Log "WARNING: Cannot send alert email (no mutt / mail present) !!!" - return 1 - fi -} - -function LoadConfigFile -{ - if [ ! -f "$1" ] - then - LogError "Cannot load backup configuration file [$1]. Backup cannot start." - return 1 - elif [[ $1 != *.conf ]] - then - LogError "Wrong configuration file supplied [$1]. Backup cannot start." - else - egrep '^#|^[^ ]*=[^;&]*' "$1" > "/dev/shm/obackup_config_$SCRIPT_PID" - source "/dev/shm/obackup_config_$SCRIPT_PID" - fi -} - -function CheckEnvironment -{ - sed --version > /dev/null 2>&1 - if [ $? != 0 ] - then - LogError "GNU coreutils not found (tested for sed --version). Backup cannot start." - return 1 - fi - - - if [ "$REMOTE_BACKUP" == "yes" ] - then - if ! type -p ssh > /dev/null 2>&1 - then - LogError "ssh not present. Cannot start backup." - return 1 - fi - - if [ "$BACKUP_SQL" != "no" ] - then - if ! type -p mysqldump > /dev/null 2>&1 - then - LogError "mysqldump not present. Cannot start backup." - return 1 - fi - fi - fi - - if [ "$BACKUP_FILES" != "no" ] - then - if ! type -p rsync > /dev/null 2>&1 - then - LogError "rsync not present. Backup cannot start." - return 1 - fi - fi -} - -# Waits for pid $1 to complete. Will log an alert if $2 seconds exec time exceeded unless $2 equals 0. Will stop task and log alert if $3 seconds exec time exceeded. -function WaitForTaskCompletition -{ - soft_alert=0 - SECONDS_BEGIN=$SECONDS - while ps -p$1 > /dev/null - do - Spinner - sleep 1 - EXEC_TIME=$(($SECONDS - $SECONDS_BEGIN)) - if [ $(($EXEC_TIME % $KEEP_LOGGING)) -eq 0 ] - then - Log "Current task still running." - fi - if [ $EXEC_TIME -gt $2 ] - then - if [ $soft_alert -eq 0 ] && [ $2 != 0 ] - then - LogError "Max soft execution time exceeded for task." - soft_alert=1 - fi - if [ $EXEC_TIME -gt $3 ] && [ $3 != 0 ] - then - LogError "Max hard execution time exceeded for task. Stopping task execution." - return 1 - fi - fi - done -} - - -## Runs local command $1 and waits for completition in $2 seconds -function RunLocalCommand -{ - CheckConnectivity3rdPartyHosts - $1 > /dev/shm/obackup_run_local_$SCRIPT_PID & - child_pid=$! - WaitForTaskCompletition $child_pid 0 $2 - wait $child_pid - retval=$? - if [ $retval -eq 0 ] - then - Log "Running command [$1] on local host succeded." - else - Log "Running command [$1] on local host failed." - fi - - Log "Command output:" - Log "$(cat /dev/shm/obackup_run_local_$SCRIPT_PID)" -} - -## Runs remote command $1 and waits for completition in $2 seconds -function RunRemoteCommand -{ - CheckConnectivity3rdPartyHosts - if [ "$REMOTE_BACKUP" == "yes" ] - then - CheckConnectivityRemoteHost - if [ $? != 0 ] - then - LogError "Connectivity test failed. Cannot run remote command." - return 1 - else - $(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT "$1" > /dev/shm/obackup_run_remote_$SCRIPT_PID & - fi - child_pid=$! - WaitForTaskCompletition $child_pid 0 $2 - wait $child_pid - retval=$? - if [ $retval -eq 0 ] - then - Log "Running command [$1] succeded." - else - LogError "Running command [$1] failed." - fi - - if [ -f /dev/shm/obackup_run_remote_$SCRIPT_PID ] - then - Log "Command output: $(cat /dev/shm/obackup_run_remote_$SCRIPT_PID)" - fi - fi -} - -function RunBeforeHook -{ - if [ "$LOCAL_RUN_BEFORE_CMD" != "" ] - then - RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE - fi - - if [ "$REMOTE_RUN_BEFORE_CMD" != "" ] - then - RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE - fi -} - -function RunAfterHook -{ - if [ "$LOCAL_RUN_AFTER_CMD" != "" ] - then - RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER - fi - - if [ "$REMOTE_RUN_AFTER_CMD" != "" ] - then - RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER - fi -} - -function SetCompressionOptions -{ - if [ "$COMPRESSION_PROGRAM" == "xz" ] && type -p xz > /dev/null 2>&1 - then - COMPRESSION_EXTENSION=.xz - elif [ "$COMPRESSION_PROGRAM" == "lzma" ] && type -p lzma > /dev/null 2>&1 - then - COMPRESSION_EXTENSION=.lzma - elif [ "$COMPRESSION_PROGRAM" == "gzip" ] && type -p gzip > /dev/null 2>&1 - then - COMPRESSION_EXTENSION=.gz - COMPRESSION_OPTIONS=--rsyncable - else - COMPRESSION_EXTENSION= - fi - - if [ "$SSH_COMPRESSION" == "yes" ] - then - SSH_COMP=-C - else - SSH_COMP= - fi -} - -function SetSudoOptions -{ - ## Add this to support prior config files without RSYNC_EXECUTABLE option - if [ "$RSYNC_EXECUTABLE" == "" ] - then - RSYNC_EXECUTABLE=rsync - fi - - if [ "$SUDO_EXEC" == "yes" ] - then - RSYNC_PATH="sudo $(which $RSYNC_EXECUTABLE)" - COMMAND_SUDO="sudo" - else - RSYNC_PATH="$(which $RSYNC_EXECUTABLE)" - COMMAND_SUDO="" - fi -} - -function CreateLocalStorageDirectories -{ - if [ ! -d $LOCAL_SQL_STORAGE ] && [ "$BACKUP_SQL" != "no" ] - then - mkdir -p $LOCAL_SQL_STORAGE - fi - - if [ ! -d $LOCAL_FILE_STORAGE ] && [ "$BACKUP_FILES" != "no" ] - then - mkdir -p $LOCAL_FILE_STORAGE - fi -} - -function CheckSpaceRequirements -{ - if [ "$BACKUP_SQL" != "no" ] - then - if [ -d $LOCAL_SQL_STORAGE ] - then - # Not elegant solution to make df silent on errors - df -P $LOCAL_SQL_STORAGE > /dev/shm/obackup_local_sql_storage_$SCRIPT_PID 2>&1 - if [ $? != 0 ] - then - LOCAL_SQL_SPACE=0 - else - LOCAL_SQL_SPACE=$(cat /dev/shm/obackup_local_sql_storage_$SCRIPT_PID | tail -1 | awk '{print $4}') - LOCAL_SQL_DRIVE=$(cat /dev/shm/obackup_local_sql_storage_$SCRIPT_PID | tail -1 | awk '{print $1}') - fi - - if [ $LOCAL_SQL_SPACE -eq 0 ] - then - LogError "Local sql storage space reported to be 0Ko." - elif [ $LOCAL_SQL_SPACE -lt $TOTAL_DATABASES_SIZE ] - then - LogError "Local disk space may be insufficient to backup files (available space is lower than non compressed databases)." - fi - else - LOCAL_SQL_SPACE=0 - LogError "SQL storage path [$LOCAL_SQL_STORAGE] doesn't exist." - fi - fi - - if [ "$BACKUP_FILES" != "no" ] - then - if [ -d $LOCAL_FILE_STORAGE ] - then - df -P $LOCAL_FILE_STORAGE > /dev/shm/obackup_local_file_storage_$SCRIPT_PID 2>&1 - if [ $? != 0 ] - then - LOCAL_FILE_SPACE=0 - else - LOCAL_FILE_SPACE=$(cat /dev/shm/obackup_local_file_storage_$SCRIPT_PID | tail -1 | awk '{print $4}') - LOCAL_FILE_DRIVE=$(cat /dev/shm/obackup_local_file_storage_$SCRIPT_PID | tail -1 | awk '{print $1}') - fi - - if [ $LOCAL_FILE_SPACE -eq 0 ] - then - LogError "Local file storage space reported to be 0Ko." - elif [ $LOCAL_FILE_SPACE -lt $TOTAL_FILES_SIZE ] - then - LogError "Local disk space may be insufficient to backup files (available space is lower than full backup)." - fi - else - LOCAL_FILE_SPACE=0 - LogError "File storage path [$LOCAL_FILE_STORAGE] doesn't exist." - fi - fi - - if [ "$LOCAL_SQL_DRIVE" == "$LOCAL_FILE_DRIVE" ] - then - LOCAL_SPACE=$LOCAL_FILE_SPACE - else - LOCAL_SPACE=$(($LOCAL_SQL_SPACE+$LOCAL_FILE_SPACE)) - fi - - if [ $BACKUP_SIZE_MINIMUM -gt $(($TOTAL_DATABASES_SIZE+$TOTAL_FILES_SIZE)) ] - then - LogError "Backup size is smaller then expected." - elif [ $LOCAL_STORAGE_WARN_MIN_SPACE -gt $LOCAL_SPACE ] - then - LogError "Local disk space is lower than warning value [$LOCAL_STORAGE_WARN_MIN_SPACE Ko]." - fi - Log "Local Space: $LOCAL_SPACE Ko - Databases size: $TOTAL_DATABASES_SIZE Ko - Files size: $TOTAL_FILES_SIZE Ko" -} - -function CheckTotalExecutionTime -{ - #### Check if max execution time of whole script as been reached - if [ $SECONDS -gt $SOFT_MAX_EXEC_TIME_TOTAL ] - then - if [ $soft_alert_total -eq 0 ] - then - LogError "Max soft execution time of the whole backup exceeded while backing up $BACKUP_TASK." - soft_alert_total=1 - fi - if [ $SECONDS -gt $HARD_MAX_EXEC_TIME_TOTAL ] - then - LogError "Max hard execution time of the whole backup exceeded while backing up $BACKUP_TASK, stopping backup process." - exit 1 - fi - fi -} - -function CheckConnectivityRemoteHost -{ - if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_BACKUP" != "no" ] - then - ping $REMOTE_HOST -c 2 > /dev/null 2>&1 - if [ $? != 0 ] - then - LogError "Cannot ping $REMOTE_HOST" - return 1 - fi - fi -} - -function CheckConnectivity3rdPartyHosts -{ - if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ] - then - remote_3rd_party_success=0 - for $i in $REMOTE_3RD_PARTY_HOSTS - do - ping $i -c 2 > /dev/null 2>&1 - if [ $? != 0 ] - then - LogError "Cannot ping 3rd party host $i" - else - remote_3rd_party_success=1 - fi - done - if [ $remote_3rd_party_success -ne 1 ] - then - LogError "No remote 3rd party host responded to ping. No internet ?" - return 1 - fi - fi -} - -function ListDatabases -{ - SECONDS_BEGIN=$SECONDS - Log "Listing databases." - CheckConnectivity3rdPartyHosts - if [ "$REMOTE_BACKUP" == "yes" ] - then - CheckConnectivityRemoteHost - if [ $? != 0 ] - then - LogError "Connectivity test failed. Stopping current task." - Dummy & - else - $(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT "mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'" > /dev/shm/obackup_dblist_$SCRIPT_PID & - fi - else - mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > /dev/shm/obackup_dblist_$SCRIPT_PID & - fi - child_pid=$! - WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK - wait $child_pid - retval=$? - if [ $retval -eq 0 ] - then - Log "Listing databases succeeded." - else - LogError "Listing databases failed." - if [ -f /dev/shm/obackup_dblist_$SCRIPT_PID ] - then - LogError "Command output: $(cat /dev/shm/obackup_dblist_$SCRIPT_PID)" - fi - return $retval - fi - - while read line - do - db_name=$(echo $line | cut -d' ' -f1) - db_size=$(echo $line | cut -d' ' -f2) - - if [ "$DATABASES_ALL" == "yes" ] - then - db_backup=1 - for j in $DATABASES_ALL_EXCLUDE_LIST - do - if [ "$db_name" == "$j" ] - then - db_backup=0 - fi - done - else - db_backup=0 - for j in $DATABASES_LIST - do - if [ "$db_name" == "$j" ] - then - db_backup=1 - fi - done - fi - - if [ $db_backup -eq 1 ] - then - if [ "$DATABASES_TO_BACKUP" != "" ] - then - DATABASES_TO_BACKUP="$DATABASES_TO_BACKUP $db_name" - else - DATABASES_TO_BACKUP=$db_name - fi - TOTAL_DATABASES_SIZE=$((TOTAL_DATABASES_SIZE+$db_size)) - else - DATABASES_EXCLUDED_LIST="$DATABASES_EXCLUDED_LIST $db_name" - fi - done < <(cat /dev/shm/obackup_dblist_$SCRIPT_PID) - return 0 -} - -function BackupDatabase -{ - CheckConnectivity3rdPartyHosts - if [ "$REMOTE_BACKUP" == "yes" ] && [ "$COMPRESSION_REMOTE" == "no" ] - then - CheckConnectivityRemoteHost - if [ $? != 0 ] - then - LogError "Connectivity test failed. Stopping current task." - exit 1 - fi - $(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT mysqldump -u $SQL_USER --skip-lock-tables --single-transaction --database $1 | $COMPRESSION_PROGRAM -$COMPRESSION_LEVEL $COMPRESSION_OPTIONS > $LOCAL_SQL_STORAGE/$1.sql$COMPRESSION_EXTENSION - elif [ "$REMOTE_BACKUP" == "yes" ] && [ "$COMPRESSION_REMOTE" == "yes" ] - then - CheckConnectivityRemoteHost - if [ $? != 0 ] - then - LogError "Connectivity test failed. Stopping current task." - exit 1 - fi - $(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT "mysqldump -u $SQL_USER --skip-lock-tables --single-transaction --database $1 | $COMPRESSION_PROGRAM -$COMPRESSION_LEVEL $COMPRESSION_OPTIONS" > $LOCAL_SQL_STORAGE/$1.sql$COMPRESSION_EXTENSION - else - mysqldump -u $SQL_USER --skip-lock-tables --single-transaction --database $1 | $COMPRESSION_PROGRAM -$COMPRESSION_LEVEL $COMPRESSION_OPTIONS > $LOCAL_SQL_STORAGE/$1.sql$COMPRESSION_EXTENSION - fi - exit $? -} - -function BackupDatabases -{ - for BACKUP_TASK in $DATABASES_TO_BACKUP - do - Log "Backing up database $BACKUP_TASK" - SECONDS_BEGIN=$SECONDS - BackupDatabase $BACKUP_TASK & - child_pid=$! - WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK - wait $child_pid - retval=$? - if [ $retval -ne 0 ] - then - LogError "Backup failed." - else - Log "Backup succeeded." - fi - - CheckTotalExecutionTime - done -} - -# Fetches single quoted directory listing including recursive ones separated by commas (eg '/dir1';'/dir2';'/dir3') -function ListDirectories -{ - SECONDS_BEGIN=$SECONDS - Log "Listing directories to backup." - OLD_IFS=$IFS - IFS=$PATH_SEPARATOR_CHAR - for i in $DIRECTORIES_RECURSE_LIST - do - CheckConnectivity3rdPartyHosts - if [ "$REMOTE_BACKUP" == "yes" ] - then - CheckConnectivityRemoteHost - if [ $? != 0 ] - then - LogError "Connectivity test failed. Stopping current task." - Dummy & - else - $(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT "$COMMAND_SUDO find $i/ -mindepth 1 -maxdepth 1 -type d" > /dev/shm/obackup_dirs_recurse_list_$SCRIPT_PID & - fi - else - $COMMAND_SUDO find $i/ -mindepth 1 -maxdepth 1 -type d > /dev/shm/obackup_dirs_recurse_list_$SCRIPT_PID & - fi - child_pid=$! - WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK - wait $child_pid - retval=$? - if [ $retval != 0 ] - then - LogError "Could not enumerate recursive directories in $i." - if [ -f /dev/shm/obackup_dirs_recurse_list_$SCRIPT_PID ] - then - LogError "Command output: $(cat /dev/shm/obackup_dirs_recurse_list_$SCRIPT_PID)" - fi - return 1 - else - Log "Listing of recursive directories succeeded for $i." - fi - - while read line - do - file_exclude=0 - for k in $DIRECTORIES_RECURSE_EXCLUDE_LIST - do - if [ "$k" == "$line" ] - then - file_exclude=1 - fi - done - - if [ $file_exclude -eq 0 ] - then - if [ "$DIRECTORIES_TO_BACKUP" == "" ] - then - DIRECTORIES_TO_BACKUP="'$line'" - else - DIRECTORIES_TO_BACKUP="$DIRECTORIES_TO_BACKUP$PATH_SEPARATOR_CHAR'$line'" - fi - else - DIRECTORIES_EXCLUDED_LIST="$DIRECTORIES_EXCLUDED_LIST$PATH_SEPARATOR_CHAR'$line'" - fi - done < <(cat /dev/shm/obackup_dirs_recurse_list_$SCRIPT_PID) - done - DIRECTORIES_TO_BACKUP_RECURSE=$DIRECTORIES_TO_BACKUP - - for i in $DIRECTORIES_SIMPLE_LIST - do - if [ "$DIRECTORIES_TO_BACKUP" == "" ] - then - DIRECTORIES_TO_BACKUP="'$i'" - else - DIRECTORIES_TO_BACKUP="$DIRECTORIES_TO_BACKUP$PATH_SEPARATOR_CHAR'$i'" - fi - done - - IFS=$OLD_IFS -} - -function GetDirectoriesSize -{ - # remove the path separator char from the dir list with sed 's/;/ /g' - dir_list=$(echo $DIRECTORIES_TO_BACKUP | sed 's/'"$PATH_SEPARATOR_CHAR"'/ /g' ) - Log "Getting files size" - CheckConnectivity3rdPartyHosts - if [ "$REMOTE_BACKUP" == "yes" ] - then - CheckConnectivityRemoteHost - if [ $? != 0 ] - then - LogError "Connectivity test failed. Stopping current task." - Dummy & - else - $(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT "echo $dir_list | xargs $COMMAND_SUDO du -cs | tail -n1 | cut -f1" > /dev/shm/obackup_fsize_$SCRIPT_PID & - fi - else - echo $dir_list | xargs $COMMAND_SUDO du -cs | tail -n1 | cut -f1 > /dev/shm/obackup_fsize_$SCRIPT_PID & - fi - child_pid=$! - WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK - wait $child_pid - retval=$? - if [ $retval != 0 ] - then - LogError "Could not get files size." - if [ -f /dev/shm/obackup_fsize_$SCRIPT_PID ] - then - LogError "Command output: $(cat /dev/shm/obackup_fsize_$SCRIPT_PID)" - fi - return 1 - else - Log "File size fetched successfully." - TOTAL_FILES_SIZE=$(cat /dev/shm/obackup_fsize_$SCRIPT_PID) - fi -} - -function RsyncExcludePattern -{ - OLD_IFS=$IFS - IFS=$PATH_SEPARATOR_CHAR - for excludedir in $RSYNC_EXCLUDE_PATTERN - do - if [ "$RSYNC_EXCLUDE" == "" ] - then - RSYNC_EXCLUDE="--exclude=$(EscapeSpaces $excludedir)" - else - RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=$(EscapeSpaces $excludedir)" - fi - done - IFS=$OLD_IFS -} - -function RsyncArgs -{ - RSYNC_ARGS=-rlptgoDE - if [ "$PRESERVE_ACLS" == "yes" ] - then - RSYNC_ARGS=$RSYNC_ARGS"A" - fi - - if [ "$PRESERVE_XATTR" == "yes" ] - then - RSYNC_ARGS=$RSYNC_ARGS"X" - fi - - if [ "$RSYNC_COMPRESS" == "yes" ] - then - RSYNC_ARGS=$RSYNC_ARGS"z" - fi -} - -function Rsync -{ - i="$(StripQuotes $1)" - if [ "$LOCAL_STORAGE_KEEP_ABSOLUTE_PATHS" == "yes" ] - then - local_file_storage_path="$(dirname $LOCAL_FILE_STORAGE$i)" - else - #### Leave the last directory path if recursive task when absolute paths not set so paths won't be mixed up - if [ "$2" == "recurse" ] - then - local_file_storage_path="$LOCAL_FILE_STORAGE/$(basename $(dirname $i))" - else - local_file_storage_path="$LOCAL_FILE_STORAGE" - fi - fi - if [ ! -d $local_file_storage_path ] - then - mkdir -p "$local_file_storage_path" - fi - - CheckConnectivity3rdPartyHosts - if [ "$REMOTE_BACKUP" == "yes" ] - then - CheckConnectivityRemoteHost - if [ $? != 0 ] - then - LogError "Connectivity test failed. Stopping current task." - exit 1 - fi - rsync_cmd="$(which $RSYNC_EXECUTABLE) $RSYNC_ARGS --delete $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -p $REMOTE_PORT\" \"$REMOTE_USER@$REMOTE_HOST:$1\" \"$local_file_storage_path\" > /dev/shm/obackup_rsync_output_$SCRIPT_PID 2>&1" - else - rsync_cmd="$(which $RSYNC_EXECUTABLE) $RSYNC_ARGS --delete $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$1\" \"$local_file_storage_path\" > /dev/shm/obackup_rsync_output_$SCRIPT_PID 2>&1" - fi - #### Eval is used so the full command is processed without bash adding single quotes round variables - if [ "$DEBUG" == "yes" ] - then - Log $rsync_cmd - fi - eval $rsync_cmd - exit $? -} - -#### First backup simple list then recursive list -function FilesBackup -{ - OLD_IFS=$IFS - IFS=$PATH_SEPARATOR_CHAR - for BACKUP_TASK in $DIRECTORIES_SIMPLE_LIST - do - BACKUP_TASK=$(StripQuotes $BACKUP_TASK) - Log "Beginning file backup $BACKUP_TASK" - SECONDS_BEGIN=$SECONDS - Rsync $BACKUP_TASK & - child_pid=$! - WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK - wait $child_pid - retval=$? - if [ $retval -ne 0 ] - then - LogError "Backup failed on remote files." - if [ -f /dev/shm/obackup_rsync_output_$SCRIPT_PID ] - then - LogError "$(cat /dev/shm/obackup_rsync_output_$SCRIPT_PID)" - fi - else - Log "Backup succeeded." - fi - CheckTotalExecutionTime - done - - for BACKUP_TASK in $DIRECTORIES_TO_BACKUP_RECURSE - do - BACKUP_TASK=$(StripQuotes $BACKUP_TASK) - Log "Beginning file backup $BACKUP_TASK" - SECONDS_BEGIN=$SECONDS - Rsync $BACKUP_TASK "recurse" & - child_pid=$! - WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK - wait $child_pid - retval=$? - if [ $retval -ne 0 ] - then - LogError "Backup failed on remote files." - if [ -f /dev/shm/obackup_rsync_output_$SCRIPT_PID ] - then - LogError "$(cat /dev/shm/obackup_rsync_output_$SCRIPT_PID)" - fi - else - Log "Backup succeeded." - fi - CheckTotalExecutionTime - done - IFS=$OLD_IFS -} - -# Will rotate everything in $1 -function RotateBackups -{ - for backup in $(ls -I "*.obackup.*" $1) - do - copy=$ROTATE_COPIES - while [ $copy -gt 1 ] - do - if [ $copy -eq $ROTATE_COPIES ] - then - rm -rf "$1/$backup.obackup.$copy" - fi - path="$1/$backup.obackup.$(($copy-1))" - if [[ -f $path || -d $path ]] - then - mv $path "$1/$backup.obackup.$copy" - fi - copy=$(($copy-1)) - done - - # Latest file backup will not be moved if script configured for remote backup so next rsync execution will only do delta copy instead of full one - if [[ $backup == *.sql.* ]] - then - mv "$1/$backup" "$1/$backup.obackup.1" - elif [ "$REMOTE_BACKUP" == "yes" ] - then - cp -R "$1/$backup" "$1/$backup.obackup.1" - else - mv "$1/$backup" "$1/$backup.obackup.1" - fi - done -} - -function Init -{ - # Set error exit code if a piped command fails - set -o pipefail - set -o errtrace - - trap TrapStop SIGINT SIGQUIT - if [ "$DEBUG" == "yes" ] - then - trap 'TrapError ${LINENO} $?' ERR - fi - - LOG_FILE=/var/log/obackup_$OBACKUP_VERSION-$BACKUP_ID.log - MAIL_ALERT_MSG="Warning: Execution of obackup instance $BACKUP_ID (pid $SCRIPT_PID) as $LOCAL_USER@$LOCAL_HOST produced errors." - -} - -function DryRun -{ - Log "/!\ DRY RUN as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" - - SetCompressionOptions - SetSudoOptions - - if [ "$BACKUP_SQL" != "no" ] - then - ListDatabases - fi - if [ "$BACKUP_FILES" != "no" ] - then - ListDirectories - GetDirectoriesSize - fi - - Log "DB backup list: $DATABASES_TO_BACKUP" - Log "DB exclude list: $DATABASES_EXCLUDED_LIST" - Log "Dirs backup list: $DIRECTORIES_TO_BACKUP" - Log "Dirs exclude list: $DIRECTORIES_EXCLUDED_LIST" - - CheckSpaceRequirements -} - -function Main -{ - Log "Backup launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" - - SetCompressionOptions - SetSudoOptions - - if [ "$BACKUP_SQL" != "no" ] - then - ListDatabases - fi - if [ "$BACKUP_FILES" != "no" ] - then - ListDirectories - GetDirectoriesSize - fi - CreateLocalStorageDirectories - CheckSpaceRequirements - - # Make Backup - if [ "$BACKUP_SQL" != "no" ] - then - if [ "$ROTATE_BACKUPS" == "yes" ] - then - RotateBackups $LOCAL_SQL_STORAGE - fi - BackupDatabases - fi - if [ "$BACKUP_FILES" != "no" ] - then - if [ "$ROTATE_BACKUPS" == "yes" ] - then - RotateBackups $LOCAL_FILE_STORAGE - fi - RsyncExcludePattern - RsyncArgs - FilesBackup - fi - # Be a happy sysadmin (and drink a coffee ? Nahh... it's past midnight.) -} - -function Usage -{ - echo "Obackup $OBACKUP_VERSION $OBACKUP_BUILD" - echo "" - echo "usage: obackup backup_name [--dry] [--silent]" - echo "" - echo "--dry: will run obackup without actually doing anything, just testing" - echo "--silent: will run obackup without any output to stdout, usefull for cron backups" - exit 128 -} - -# Command line argument flags -dryrun=0 -silent=0 -# Alert flags -soft_alert_total=0 -error_alert=0 - -if [ $# -eq 0 ] -then - Usage -fi - -for i in "$@" -do - case $i in - --dry) - dryrun=1 - ;; - --silent) - silent=1 - ;; - --help|-h) - Usage - ;; - esac -done - -CheckEnvironment -if [ $? == 0 ] -then - if [ "$1" != "" ] - then - LoadConfigFile $1 - if [ $? == 0 ] - then - Init - DATE=$(date) - Log "--------------------------------------------------------------------" - Log "$DATE - Obackup v$OBACKUP_VERSION script begin." - Log "--------------------------------------------------------------------" - if [ $dryrun -eq 1 ] - then - DryRun - else - OLD_IFS=$IFS - RunBeforeHook - Main - IFS=$OLD_IFS - RunAfterHook - fi - CleanUp - else - LogError "Configuration file could not be loaded." - exit 1 - fi - else - LogError "No configuration file provided." - exit 1 - fi -else - LogError "Environment not suitable to run obackup." -fi - -if [ $error_alert -ne 0 ] -then - SendAlert - LogError "Backup script finished with errors." - exit 1 -else - Log "Backup script finshed." - exit 0 -fi