mirror of https://github.com/deajan/obackup.git
Initial public release.
This commit is contained in:
deajan
commit
40edd19e04
|
|
@ -0,0 +1,80 @@
|
|||
#!/bin/bash
|
||||
|
||||
###### Remote (or local) backup script for files & databases
|
||||
###### (L) 2013 by Ozy de Jong (www.badministrateur.com)
|
||||
|
||||
## Backup identification, any string you want
|
||||
BACKUP_ID="your backup identification string (eg: hostname)"
|
||||
|
||||
## General backup options
|
||||
BACKUP_SQL=yes
|
||||
BACKUP_FILES=yes
|
||||
|
||||
## Local storage paths
|
||||
LOCAL_SQL_STORAGE="/home/storage/backup/sql"
|
||||
LOCAL_FILE_STORAGE="/home/storage/backup/files"
|
||||
## Keep the absolute source path in your backup, eg: /your/backup/storage/the/remote/server/files
|
||||
## You should leave this enabled if you use recursive directories backup lists or they'll all end in the same path.
|
||||
LOCAL_STORAGE_KEEP_ABSOLUTE_PATHS=yes
|
||||
## Generate an alert if backup size is lower than given value in Kb, useful for local mounted backups.
|
||||
BACKUP_SIZE_MINIMUM=1024
|
||||
## Generate an alert if local storage free space is lower than given value in Kb.
|
||||
LOCAL_STORAGE_WARN_MIN_SPACE=1048576
|
||||
|
||||
## If enabled, file backups will be processed with sudo command. See documentation for /etc/sudoers configuration ("find", "du" and "rsync" need to be allowed). Requiretty needs to be disabled.
|
||||
SUDO_EXEC=yes
|
||||
|
||||
## Remote options (will make backups of remote system through ssh tunnel, public RSA key need to be put into /home/.ssh/authorized_keys in remote users home directory)
|
||||
REMOTE_BACKUP=yes
|
||||
SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa
|
||||
REMOTE_USER=backupuser
|
||||
REMOTE_HOST=yourhost.local
|
||||
REMOTE_PORT=22
|
||||
## ssh compression should be used unless your remote connection is good enough (LAN)
|
||||
SSH_COMPRESSION=yes
|
||||
## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
|
||||
REMOTE_HOST_PING=yes
|
||||
## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
|
||||
REMOTE_3RD_PARTY_HOST="www.kernel.org"
|
||||
|
||||
## Databases options
|
||||
SQL_USER=backupuser
|
||||
## Save all databases except the ones specified in the exlude list. Every found database will be backed up as separate task (see documentation for explanation about tasks)
|
||||
DATABASES_ALL=yes
|
||||
DATABASES_ALL_EXCLUDE_LIST="test"
|
||||
# Alternatively, you can specifiy a manual list of databases to backup separated by spaces
|
||||
DATABASES_LIST=""
|
||||
## Max backup execution time per DB task. Soft is warning only. Hard is warning, stopping backup task and processing next one. Time is specified in seconds
|
||||
SOFT_MAX_EXEC_TIME_DB_TASK=3600
|
||||
HARD_MAX_EXEC_TIME_DB_TASK=7200
|
||||
## Preferred sql dump compression. Can be set to xz, lzma or gzip.
|
||||
## Generally, xz level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable.
|
||||
COMPRESSION_PROGRAM=xz
|
||||
COMPRESSION_LEVEL=3
|
||||
## Dump compression should be done on remote side but can also be done locally to lower remote system usage (will take more bandwidth, check for ssh compression)
|
||||
COMPRESSION_REMOTE=yes
|
||||
|
||||
## Path separator. You can set whatever seperator you want in your directories list below. You may change this in case you have some esoteric filenames (try to use unconventional separators like | ).
|
||||
PATH_SEPARATOR_CHAR=";"
|
||||
## File backup lists. Simple double quoted directory list separated by the $PATH_SEPARATOR_CHAR. Every directory will be processed as task (see documentation for explanation about tasks)
|
||||
DIRECTORIES_SIMPLE_LIST="/var/named"
|
||||
## Recurse directory list separated by the $PATH_SEPARATOR_CHAR. Will create a backup task per subdirectory (one level only), eg RECURSE_LIST="/home /var" will create tasks "/home/dir1", "/home/dir2", ... "/home/dirN", "/var/log", "/var/lib"... "/var/whatever"
|
||||
DIRECTORIES_RECURSE_LIST="/home"
|
||||
## You can optionally exclude directories from RECURSE_LIST tasks, eg on the above example you could exclude /home/dir2 by adding it to RECURSE_EXCLUDE_LIST
|
||||
DIRECTORIES_RECURSE_EXCLUDE_LIST="/home/backupuser;/home/lost+found"
|
||||
# Be aware that every recurse list will have it's own root (exclude pattern is relative from /home/web for /home/web/{recursedir})
|
||||
RSYNC_EXCLUDE_PATTERN="*/tmp;*/ftp/www/cache/cachefs;*/sessions"
|
||||
## Max execution time per file backup task. Soft is warning only. Hard is warning, stopping backup and processing next one one file list. Tilme is specified in seconds
|
||||
SOFT_MAX_EXEC_TIME_FILE_TASK=3600
|
||||
HARD_MAX_EXEC_TIME_FILE_TASK=7200
|
||||
|
||||
## Alert email adresses separated by a space character
|
||||
DESTINATION_MAIL="your@mail.address"
|
||||
|
||||
## Max execution time of whole backup process. Soft is warning only. Hard is warning and stopping whole backup process.
|
||||
SOFT_MAX_EXEC_TIME_TOTAL=30000
|
||||
HARD_MAX_EXEC_TIME_TOTAL=36000
|
||||
|
||||
## Backup Rotation in case you don't use a snapshot aware file system like zfs or btrfs to perform a snapshot before every backup
|
||||
ROTATE_BACKUPS=no
|
||||
ROTATE_COPIES=7
|
||||
|
|
@ -0,0 +1,857 @@
|
|||
#!/bin/bash
|
||||
|
||||
###### Remote (or local) backup script for files & databases
|
||||
###### (L) 2013 by Ozy de Jong (www.badministrateur.com)
|
||||
OBACKUP_VERSION=1.83 #### Build 3005201301
|
||||
|
||||
LOG_FILE=/var/log/obackup_$OBACKUP_VERSION-$BACKUP_ID.log
|
||||
DEBUG=no
|
||||
SCRIPT_PID=$$
|
||||
|
||||
LOCAL_USER=$(whoami)
|
||||
LOCAL_HOST=$(hostname)
|
||||
MAIL_ALERT_MSG="Warning: Execution of obackup instance $BACKUP_ID (pid $SCRIPT_PID) as $LOCAL_USER@$LOCAL_HOST produced errors."
|
||||
|
||||
## Log a state message every $KEEP_LOGGING seconds. Should generally not be equal to soft or hard execution time so your log won't be unnecessary big.
|
||||
KEEP_LOGGING=1801
|
||||
|
||||
## Global variables and forked command results
|
||||
DATABASES_TO_BACKUP="" # Processed list of DBs that will be backed up
|
||||
DATABASES_EXCLUDED_LIST="" # Processed list of DBs that won't be backed up
|
||||
TOTAL_DATABASES_SIZE=0 # Total DB size of $DATABASES_TO_BACKUP
|
||||
DIRECTORIES_RECURSE_TO_BACKUP="" # Processed list of recursive directories that will be backed up
|
||||
DIRECTORIES_EXCLUDED_LIST="" # Processed list of recursive directorires that won't be backed up
|
||||
DIRECTORIES_TO_BACKUP="" # Processed list of all directories to backup
|
||||
TOTAL_FILES_SIZE=0 # Total file size of $DIRECTORIES_TO_BACKUP
|
||||
|
||||
# /dev/shm/obackup_dblist_$SCRIPT_PID Databases list and sizes
|
||||
# /dev/shm/obackup_dirs_recurse_list_$SCRIPT_PID Recursive directories list
|
||||
# /dev/shm/obackup_local_space_$SCRIPT_PID Local free space
|
||||
# /dev/shm/obackup_fsize_$SCRIPT_PID Size of $DIRECTORIES_TO_BACKUP
|
||||
# /dev/shm/obackup_rsync_output_$SCRIPT_PID Output of Rsync command
|
||||
|
||||
soft_alert_total=0
|
||||
error_alert=0
|
||||
|
||||
function Log
|
||||
{
|
||||
echo "TIME: $SECONDS - $1" >> "$LOG_FILE"
|
||||
echo "TIME: $SECONDS - $1"
|
||||
}
|
||||
|
||||
function LogError
|
||||
{
|
||||
Log "$1"
|
||||
error_alert=1
|
||||
}
|
||||
|
||||
function TrapError
|
||||
{
|
||||
local JOB="$0"
|
||||
local LINE="$1"
|
||||
local CODE="${2:-1}"
|
||||
echo " /!\ Error in ${JOB}: Near line ${LINE}, exit code ${CODE}"
|
||||
}
|
||||
|
||||
function TrapStop
|
||||
{
|
||||
LogError " /!\ WARNING: Manual extit of backup script. Backups may be in inconsistent state."
|
||||
if [ "$DEBUG" == "no" ]
|
||||
then
|
||||
CleanUp
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
|
||||
function Spinner
|
||||
{
|
||||
case $toggle
|
||||
in
|
||||
1)
|
||||
echo -n $1" \ "
|
||||
echo -ne "\r"
|
||||
toggle="2"
|
||||
;;
|
||||
|
||||
2)
|
||||
echo -n $1" | "
|
||||
echo -ne "\r"
|
||||
toggle="3"
|
||||
;;
|
||||
|
||||
3)
|
||||
echo -n $1" / "
|
||||
echo -ne "\r"
|
||||
toggle="4"
|
||||
;;
|
||||
|
||||
*)
|
||||
echo -n $1" - "
|
||||
echo -ne "\r"
|
||||
toggle="1"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
function Dummy
|
||||
{
|
||||
exit 1;
|
||||
}
|
||||
|
||||
function StripQuotes
|
||||
{
|
||||
echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")
|
||||
}
|
||||
|
||||
function EscapeSpaces
|
||||
{
|
||||
echo $(echo $1 | sed 's/ /\\ /g')
|
||||
}
|
||||
|
||||
function CleanUp
|
||||
{
|
||||
rm -f /dev/shm/obackup_dblist_$SCRIPT_PID
|
||||
rm -f /dev/shm/obackup_local_space_$SCRIPT_PID
|
||||
rm -f /dev/shm/obackup_dirs_recurse_list_$SCRIPT_PID
|
||||
rm -f /dev/shm/obackup_fsize_$SCRIPT_PID
|
||||
rm -f /dev/shm/obackup_rsync_output_$SCRIPT_PID
|
||||
rm -f /dev/shm/obackup_config_$SCRIPT_PID
|
||||
}
|
||||
|
||||
function SendAlert
|
||||
{
|
||||
CheckConnectivityRemoteHost
|
||||
CheckConnectivity3rdPartyHosts
|
||||
cat $LOG_FILE | gzip -9 > /tmp/obackup_lastlog.gz
|
||||
if type -p mutt > /dev/null 2>&1
|
||||
then
|
||||
echo $MAIL_ALERT_MSG | $(which mutt) -x -s "Backup alert for $BACKUP_ID" $DESTINATION_MAIL -a /tmp/obackup_lastlog.gz
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
Log "WARNING: Cannot send alert email via $(which mutt) !!!"
|
||||
else
|
||||
Log "Sent alert mail using mutt."
|
||||
fi
|
||||
elif type -p mail > /dev/null 2>&1
|
||||
then
|
||||
echo $MAIL_ALERT_MSG | $(which mail) -a /tmp/obackup_lastlog.gz -s "Backup alert for $BACKUP_ID" $DESTINATION_MAIL
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
Log "WARNING: Cannot send alert email via $(which mail) with attachments !!!"
|
||||
echo $MAIL_ALERT_MSG | $(which mail) -s "Backup alert for $BACKUP_ID" $DESTINATION_MAIL
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
Log "WARNING: Cannot send alert email via $(which mail) without attachments !!!"
|
||||
else
|
||||
Log "Sent alert mail using mail command without attachment."
|
||||
fi
|
||||
else
|
||||
Log "Sent alert mail using mail command."
|
||||
fi
|
||||
else
|
||||
Log "WARNING: Cannot send alert email (no mutt / mail present) !!!"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function LoadConfigFile
|
||||
{
|
||||
if [ ! -f "$1" ]
|
||||
then
|
||||
LogError "Cannot load backup configuration file [$1]. Backup cannot start."
|
||||
return 1
|
||||
elif [[ $1 != *.conf ]]
|
||||
then
|
||||
LogError "Wrong configuration file supplied [$1]. Backup cannot start."
|
||||
else
|
||||
egrep '^#|^[^ ]*=[^;&]*' "$1" > "/dev/shm/obackup_config_$SCRIPT_PID"
|
||||
source $1
|
||||
fi
|
||||
}
|
||||
|
||||
function CheckEnvironment
|
||||
{
|
||||
sed --version > /dev/null 2>&1
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
LogError "GNU coreutils not found (tested for sed --version). Backup cannot start."
|
||||
return 1
|
||||
fi
|
||||
|
||||
|
||||
if [ "$REMOTE_BACKUP" == "yes" ]
|
||||
then
|
||||
if ! type -p ssh > /dev/null 2>&1
|
||||
then
|
||||
LogError "ssh not present. Cannot start backup."
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ "$BACKUP_SQL" != "no" ]
|
||||
then
|
||||
if ! type -p mysqldump > /dev/null 2>&1
|
||||
then
|
||||
LogError "mysqldump not present. Cannot start backup."
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$BACKUP_FILES" != "no" ]
|
||||
then
|
||||
if ! type -p rsync > /dev/null 2>&1
|
||||
then
|
||||
LogError "rsync not present. Backup cannot start."
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function SetCompressionOptions
|
||||
{
|
||||
if [ "$COMPRESSION_PROGRAM" == "xz" ] && type -p xz > /dev/null 2>&1
|
||||
then
|
||||
COMPRESSION_EXTENSION=.xz
|
||||
elif [ "$COMPRESSION_PROGRAM" == "lzma" ] && type -p lzma > /dev/null 2>&1
|
||||
then
|
||||
COMPRESSION_EXTENSION=.lzma
|
||||
elif [ "$COMPRESSION_PROGRAM" == "gzip" ] && type -p gzip > /dev/null 2>&1
|
||||
then
|
||||
COMPRESSION_EXTENSION=.gz
|
||||
COMPRESSION_OPTIONS=--rsyncable
|
||||
else
|
||||
COMPRESSION_EXTENSION=
|
||||
fi
|
||||
|
||||
if [ "$SSH_COMPRESSION" == "yes" ]
|
||||
then
|
||||
SSH_COMP=-C
|
||||
else
|
||||
SSH_COMP=
|
||||
fi
|
||||
}
|
||||
|
||||
function SetSudoOptions
|
||||
{
|
||||
if [ "$SUDO_EXEC" == "yes" ]
|
||||
then
|
||||
RSYNC_PATH="sudo $(which rsync)"
|
||||
COMMAND_SUDO="sudo"
|
||||
else
|
||||
RSYNC_PATH="$(which rsync)"
|
||||
COMMAND_SUDO=""
|
||||
fi
|
||||
}
|
||||
|
||||
function CreateLocalStorageDirectories
|
||||
{
|
||||
if [ ! -d $LOCAL_SQL_STORAGE ] && [ "$BACKUP_SQL" != "no" ]
|
||||
then
|
||||
mkdir -p $LOCAL_SQL_STORAGE
|
||||
fi
|
||||
|
||||
if [ ! -d $LOCAL_FILE_STORAGE ] && [ "$BACKUP_FILES" != "no" ]
|
||||
then
|
||||
mkdir -p $LOCAL_FILE_STORAGE
|
||||
fi
|
||||
}
|
||||
|
||||
function CheckLocalSpace
|
||||
{
|
||||
# Not elegant solution to make df silent on errors
|
||||
df -P $LOCAL_FILE_STORAGE > /dev/shm/obackup_local_space_$SCRIPT_PID 2>&1
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
LOCAL_SPACE=0
|
||||
else
|
||||
LOCAL_SPACE=$(cat /dev/shm/obackup_local_space_$SCRIPT_PID | tail -1 | awk '{print $4}')
|
||||
fi
|
||||
|
||||
if [ $LOCAL_SPACE -eq 0 ]
|
||||
then
|
||||
LogError "Local disk space reported to be 0 Ko. This may also happen if local storage path doesn't exist."
|
||||
elif [ $BACKUP_SIZE_MINIMUM -gt $(($TOTAL_DATABASES_SIZE+$TOTAL_FILES_SIZE)) ]
|
||||
then
|
||||
LogError "Backup size is smaller then expected."
|
||||
elif [ $LOCAL_STORAGE_WARN_MIN_SPACE -gt $LOCAL_SPACE ]
|
||||
then
|
||||
LogError "Local disk space is lower than warning value ($LOCAL_SPACE free Ko)."
|
||||
elif [ $LOCAL_SPACE -lt $(($TOTAL_DATABASES_SIZE+$TOTAL_FILES_SIZE)) ]
|
||||
then
|
||||
LogError "Local disk space may be insufficient (depending on rsync delta and DB compression ratio)."
|
||||
fi
|
||||
Log "Local Space: $LOCAL_SPACE Ko - Databases size: $TOTAL_DATABASES_SIZE Ko - Files size: $TOTAL_FILES_SIZE Ko"
|
||||
}
|
||||
|
||||
# Waits for pid $1 to complete. Will log an alert if $2 seconds exec time exceeded. Will stop task and log alert if $3 seconds exec time exceeded.
|
||||
function WaitForTaskCompletition
|
||||
{
|
||||
soft_alert=0
|
||||
SECONDS_BEGIN=$SECONDS
|
||||
while ps -p$1 > /dev/null
|
||||
do
|
||||
Spinner
|
||||
sleep 1
|
||||
EXEC_TIME=$(($SECONDS - $SECONDS_BEGIN))
|
||||
if [ $(($EXEC_TIME % $KEEP_LOGGING)) -eq 0 ]
|
||||
then
|
||||
Log "Current task still running."
|
||||
fi
|
||||
if [ $EXEC_TIME -gt $2 ]
|
||||
then
|
||||
if [ $soft_alert -eq 0 ]
|
||||
then
|
||||
LogError "Max soft execution time exceeded for task."
|
||||
soft_alert=1
|
||||
fi
|
||||
if [ $EXEC_TIME -gt $3 ]
|
||||
then
|
||||
LogError "Max hard execution time exceeded for task. Stopping task execution."
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function CheckTotalExecutionTime
|
||||
{
|
||||
#### Check if max execution time of whole script as been reached
|
||||
if [ $SECONDS -gt $SOFT_MAX_EXEC_TIME_TOTAL ]
|
||||
then
|
||||
if [ $soft_alert_total -eq 0 ]
|
||||
then
|
||||
LogError "Max soft execution time of the whole backup exceeded while backing up $BACKUP_TASK."
|
||||
soft_alert_total=1
|
||||
fi
|
||||
if [ $SECONDS -gt $HARD_MAX_EXEC_TIME_TOTAL ]
|
||||
then
|
||||
LogError "Max hard execution time of the whole backup exceeded while backing up $BACKUP_TASK, stopping backup process."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function CheckConnectivityRemoteHost
|
||||
{
|
||||
if [ "$REMOTE_HOST_PING" != "no" ]
|
||||
then
|
||||
ping $REMOTE_HOST -c 2 > /dev/null 2>&1
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
LogError "Cannot ping $REMOTE_HOST"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function CheckConnectivity3rdPartyHosts
|
||||
{
|
||||
if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]
|
||||
then
|
||||
remote_3rd_party_success=0
|
||||
for $i in $REMOTE_3RD_PARTY_HOSTS
|
||||
do
|
||||
ping $i -c 2 > /dev/null 2>&1
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
LogError "Cannot ping 3rd party host $i"
|
||||
else
|
||||
remote_3rd_party_success=1
|
||||
fi
|
||||
done
|
||||
if [ $remote_3rd_party_success -ne 1 ]
|
||||
then
|
||||
LogError "No remote 3rd party host responded to ping. No internet ?"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function ListDatabases
|
||||
{
|
||||
SECONDS_BEGIN=$SECONDS
|
||||
Log "Listing databases."
|
||||
CheckConnectivity3rdPartyHosts
|
||||
if [ "$REMOTE_BACKUP" == "yes" ]
|
||||
then
|
||||
CheckConnectivityRemoteHost
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
LogError "Connectivity test failed. Stopping current task."
|
||||
Dummy &
|
||||
else
|
||||
$(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT "mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'" > /dev/shm/obackup_dblist_$SCRIPT_PID &
|
||||
fi
|
||||
else
|
||||
mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > /dev/shm/oback_dblist_$SCRIPT_PID &
|
||||
fi
|
||||
retval=$?
|
||||
child_pid=$!
|
||||
WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK
|
||||
wait $child_pid
|
||||
retval=$?
|
||||
if [ $retval -eq 0 ]
|
||||
then
|
||||
Log "Listing databases succeeded."
|
||||
else
|
||||
LogError "Listing databases failed."
|
||||
return $retval
|
||||
fi
|
||||
|
||||
while read line
|
||||
do
|
||||
db_name=$(echo $line | cut -d' ' -f1)
|
||||
db_size=$(echo $line | cut -d' ' -f2)
|
||||
|
||||
if [ "$DATABASES_ALL" == "yes" ]
|
||||
then
|
||||
db_backup=1
|
||||
for j in $DATABASES_ALL_EXCLUDE_LIST
|
||||
do
|
||||
if [ "$db_name" == "$j" ]
|
||||
then
|
||||
db_backup=0
|
||||
fi
|
||||
done
|
||||
else
|
||||
db_backup=0
|
||||
for j in $DATABASES_LIST
|
||||
do
|
||||
if [ "$db_name" == "$j" ]
|
||||
then
|
||||
db_backup=1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ $db_backup -eq 1 ]
|
||||
then
|
||||
if [ "$DATABASES_TO_BACKUP" != "" ]
|
||||
then
|
||||
DATABASES_TO_BACKUP="$DATABASES_TO_BACKUP $db_name"
|
||||
else
|
||||
DATABASES_TO_BACKUP=$db_name
|
||||
fi
|
||||
TOTAL_DATABASES_SIZE=$((TOTAL_DATABASES_SIZE+$db_size))
|
||||
else
|
||||
DATABASES_EXCLUDED_LIST="$DATABASES_EXCLUDED_LIST $db_name"
|
||||
fi
|
||||
done < <(cat /dev/shm/obackup_dblist_$SCRIPT_PID)
|
||||
return 0
|
||||
}
|
||||
|
||||
function BackupDatabase
|
||||
{
|
||||
CheckConnectivity3rdPartyHosts
|
||||
if [ "$REMOTE_BACKUP" == "yes" ] && [ "$COMPRESSION_REMOTE" == "no" ]
|
||||
then
|
||||
CheckConnectivityRemoteHost
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
LogError "Connectivity test failed. Stopping current task."
|
||||
exit 1
|
||||
fi
|
||||
$(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT mysqldump -u $SQL_USER --skip-lock-tables --single-transaction --database $1 | $COMPRESSION_PROGRAM -$COMPRESSION_LEVEL $COMPRESSION_OPTIONS > $LOCAL_SQL_STORAGE/$1.sql$COMPRESSION_EXTENSION
|
||||
elif [ "$REMOTE_BACKUP" == "yes" ] && [ "$COMPRESSION_REMOTE" == "yes" ]
|
||||
then
|
||||
CheckConnectivityRemoteHost
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
LogError "Connectivity test failed. Stopping current task."
|
||||
exit 1
|
||||
fi
|
||||
$(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT "mysqldump -u $SQL_USER --skip-lock-tables --single-transaction --database $1 | $COMPRESSION_PROGRAM -$COMPRESSION_LEVEL $COMPRESSION_OPTIONS" > $LOCAL_SQL_STORAGE/$1.sql$COMPRESSION_EXTENSION
|
||||
else
|
||||
mysqldump -u $SQL_USER --skip-lock-tables --single-transaction --database $1 | $COMPRESSION_PROGRAM -$COMPRESSION_LEVEL $COMPRESSION_OPTIONS > $LOCAL_SQL_STORAGE/$1.sql$COMPRESSION_EXTENSION
|
||||
fi
|
||||
exit $?
|
||||
}
|
||||
|
||||
function BackupDatabases
|
||||
{
|
||||
for BACKUP_TASK in $DATABASES_TO_BACKUP
|
||||
do
|
||||
Log "Backing up database $BACKUP_TASK"
|
||||
SECONDS_BEGIN=$SECONDS
|
||||
BackupDatabase $BACKUP_TASK &
|
||||
child_pid=$!
|
||||
WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK
|
||||
wait $child_pid
|
||||
retval=$?
|
||||
SECONDS_END=$SECONDS
|
||||
EXEC_TIME=$(($SECONDS_END - $SECONDS_BEGIN))
|
||||
if [ $retval -ne 0 ]
|
||||
then
|
||||
LogError "Backup failed."
|
||||
else
|
||||
Log "Backup succeeded."
|
||||
fi
|
||||
|
||||
CheckTotalExecutionTime
|
||||
done
|
||||
}
|
||||
|
||||
# Fetches single quoted directory listing including recursive ones separated by commas (eg '/dir1';'/dir2';'/dir3')
|
||||
function ListDirectories
|
||||
{
|
||||
SECONDS_BEGIN=$SECONDS
|
||||
Log "Listing directories to backup."
|
||||
OLD_IFS=$IFS
|
||||
IFS=$PATH_SEPARATOR_CHAR
|
||||
for i in $DIRECTORIES_RECURSE_LIST
|
||||
do
|
||||
CheckConnectivity3rdPartyHosts
|
||||
if [ "$REMOTE_BACKUP" == "yes" ]
|
||||
then
|
||||
CheckConnectivityRemoteHost
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
LogError "Connectivity test failed. Stopping current task."
|
||||
Dummy &
|
||||
else
|
||||
$(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT "$COMMAND_SUDO find $i/ -mindepth 1 -maxdepth 1 -type d" > /dev/shm/obackup_dirs_recurse_list_$SCRIPT_PID &
|
||||
fi
|
||||
else
|
||||
$COMMAND_SUDO find $i/ -mindepth 1 -maxdepth 1 -type d > /dev/shm/obackup_dirs_recurse_list_$SCRIPT_PID &
|
||||
fi
|
||||
child_pid=$!
|
||||
WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK
|
||||
wait $child_pid
|
||||
retval=$?
|
||||
if [ $retval != 0 ]
|
||||
then
|
||||
LogError "Could not enumerate recursive directories in $i."
|
||||
return 1
|
||||
else
|
||||
Log "Listing of recursive directories succeeded for $i."
|
||||
fi
|
||||
|
||||
while read line
|
||||
do
|
||||
file_exclude=0
|
||||
for k in $DIRECTORIES_RECURSE_EXCLUDE_LIST
|
||||
do
|
||||
if [ "$k" == "$line" ]
|
||||
then
|
||||
file_exclude=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $file_exclude -eq 0 ]
|
||||
then
|
||||
if [ "$DIRECTORIES_TO_BACKUP" == "" ]
|
||||
then
|
||||
DIRECTORIES_TO_BACKUP="'$line'"
|
||||
else
|
||||
DIRECTORIES_TO_BACKUP="$DIRECTORIES_TO_BACKUP$PATH_SEPARATOR_CHAR'$line'"
|
||||
fi
|
||||
else
|
||||
DIRECTORIES_EXCLUDED_LIST="$DIRECTORIES_EXCLUDED_LIST$PATH_SEPARATOR_CHAR'$line'"
|
||||
fi
|
||||
done < <(cat /dev/shm/obackup_dirs_recurse_list_$SCRIPT_PID)
|
||||
done
|
||||
DIRECTORIES_TO_BACKUP_RECURSE=$DIRECTORIES_TO_BACKUP
|
||||
|
||||
for i in $DIRECTORIES_SIMPLE_LIST
|
||||
do
|
||||
if [ "$DIRECTORIES_TO_BACKUP" == "" ]
|
||||
then
|
||||
DIRECTORIES_TO_BACKUP="'$i'"
|
||||
else
|
||||
DIRECTORIES_TO_BACKUP="$DIRECTORIES_TO_BACKUP$PATH_SEPARATOR_CHAR'$i'"
|
||||
fi
|
||||
done
|
||||
|
||||
IFS=$OLD_IFS
|
||||
}
|
||||
|
||||
function GetDirectoriesSize
|
||||
{
|
||||
# remove the path separator char from the dir list with sed 's/;/ /g'
|
||||
dir_list=$(echo $DIRECTORIES_TO_BACKUP | sed 's/'"$PATH_SEPARATOR_CHAR"'/ /g' )
|
||||
Log "Getting files size"
|
||||
CheckConnectivity3rdPartyHosts
|
||||
if [ "$REMOTE_BACKUP" == "yes" ]
|
||||
then
|
||||
CheckConnectivityRemoteHost
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
LogError "Connectivity test failed. Stopping current task."
|
||||
Dummy &
|
||||
else
|
||||
$(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT "echo $dir_list | xargs $COMMAND_SUDO du -cs | tail -n1 | cut -f1" > /dev/shm/obackup_fsize_$SCRIPT_PID &
|
||||
fi
|
||||
else
|
||||
echo $dir_list | xargs $COMMAND_SUDO du -cs | tail -n1 | cut -f1 > /dev/shm/obackup_fsize_$SCRIPT_PID &
|
||||
fi
|
||||
child_pid=$!
|
||||
WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK
|
||||
wait $child_pid
|
||||
retval=$?
|
||||
if [ $retval != 0 ]
|
||||
then
|
||||
LogError "Could not get files size."
|
||||
return 1
|
||||
else
|
||||
Log "File size fetched successfully."
|
||||
TOTAL_FILES_SIZE=$(cat /dev/shm/obackup_fsize_$SCRIPT_PID)
|
||||
fi
|
||||
}
|
||||
|
||||
function RsyncExcludePattern
|
||||
{
|
||||
OLD_IFS=$IFS
|
||||
IFS=$PATH_SEPARATOR_CHAR
|
||||
for excludedir in $RSYNC_EXCLUDE_PATTERN
|
||||
do
|
||||
if [ "$RSYNC_EXCLUDE" == "" ]
|
||||
then
|
||||
RSYNC_EXCLUDE="--exclude=$(EscapeSpaces $excludedir)"
|
||||
else
|
||||
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=$(EscapeSpaces $excludedir)"
|
||||
fi
|
||||
done
|
||||
IFS=$OLD_IFS
|
||||
}
|
||||
|
||||
function Rsync
|
||||
{
|
||||
i="$(StripQuotes $1)"
|
||||
if [ "$LOCAL_STORAGE_KEEP_ABSOLUTE_PATHS" == "yes" ]
|
||||
then
|
||||
local_file_storage_path="$(dirname $LOCAL_FILE_STORAGE$i)"
|
||||
else
|
||||
#### Leave the last directory path if recursive task when absolute paths not set so paths won't be mixed up
|
||||
if [ "$2" == "recurse" ]
|
||||
then
|
||||
local_file_storage_path="$LOCAL_FILE_STORAGE/$(basename $(dirname $i))"
|
||||
else
|
||||
local_file_storage_path="$LOCAL_FILE_STORAGE"
|
||||
fi
|
||||
fi
|
||||
if [ ! -d $local_file_storage_path ]
|
||||
then
|
||||
mkdir -p "$local_file_storage_path"
|
||||
fi
|
||||
|
||||
CheckConnectivity3rdPartyHosts
|
||||
if [ "$REMOTE_BACKUP" == "yes" ]
|
||||
then
|
||||
CheckConnectivityRemoteHost
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
LogError "Connectivity test failed. Stopping current task."
|
||||
exit 1
|
||||
fi
|
||||
rsync_cmd="$(which rsync) -rlptgoDE --delete $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -p $REMOTE_PORT\" \"$REMOTE_USER@$REMOTE_HOST:$1\" \"$local_file_storage_path\" > /dev/shm/obackup_rsync_output_$SCRIPT_PID 2>&1"
|
||||
else
|
||||
rsync_cmd="$(which rsync) -rlptgoDE --delete $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$1\" \"$local_file_storage_path\" > /dev/shm/obackup_rsync_output_$SCRIPT_PID 2>&1"
|
||||
fi
|
||||
#### Eval is used so the full command is processed without bash adding single quotes round variables
|
||||
if [ "$DEBUG" == "yes" ]
|
||||
then
|
||||
Log $rsync_cmd
|
||||
fi
|
||||
eval $rsync_cmd
|
||||
exit $?
|
||||
}
|
||||
|
||||
#### First backup simple list then recursive list
|
||||
function FilesBackup
|
||||
{
|
||||
OLD_IFS=$IFS
|
||||
IFS=$PATH_SEPARATOR_CHAR
|
||||
for BACKUP_TASK in $DIRECTORIES_SIMPLE_LIST
|
||||
do
|
||||
BACKUP_TASK=$(StripQuotes $BACKUP_TASK)
|
||||
Log "Beginning file backup $BACKUP_TASK"
|
||||
SECONDS_BEGIN=$SECONDS
|
||||
Rsync $BACKUP_TASK &
|
||||
child_pid=$!
|
||||
WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK
|
||||
wait $child_pid
|
||||
retval=$?
|
||||
SECONDS_END=$SECONDS
|
||||
EXEC_TIME=$(($SECONDS_END-$SECONDS_BEGIN))
|
||||
if [ $retval -ne 0 ]
|
||||
then
|
||||
LogError "Backup failed on remote files."
|
||||
LogError "$(cat /dev/shm/obackup_rsync_output_$SCRIPT_PID)"
|
||||
else
|
||||
Log "Backup succeeded."
|
||||
fi
|
||||
CheckTotalExecutionTime
|
||||
done
|
||||
|
||||
for BACKUP_TASK in $DIRECTORIES_TO_BACKUP_RECURSE
|
||||
do
|
||||
BACKUP_TASK=$(StripQuotes $BACKUP_TASK)
|
||||
Log "Beginning file backup $BACKUP_TASK"
|
||||
SECONDS_BEGIN=$SECONDS
|
||||
Rsync $BACKUP_TASK "recurse" &
|
||||
child_pid=$!
|
||||
WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK
|
||||
wait $child_pid
|
||||
retval=$?
|
||||
SECONDS_END=$SECONDS
|
||||
EXEC_TIME=$(($SECONDS_END-$SECONDS_BEGIN))
|
||||
if [ $retval -ne 0 ]
|
||||
then
|
||||
LogError "Backup failed on remote files."
|
||||
LogError "$(cat /dev/shm/obackup_rsync_output_$SCRIPT_PID)"
|
||||
else
|
||||
Log "Backup succeeded."
|
||||
fi
|
||||
CheckTotalExecutionTime
|
||||
done
|
||||
IFS=$OLD_IFS
|
||||
}
|
||||
|
||||
# Will rotate everything in $1
|
||||
function RotateBackups
|
||||
{
|
||||
for backup in $(ls -I "*.obackup.*" $1)
|
||||
do
|
||||
copy=$ROTATE_COPIES
|
||||
while [ $copy -gt 1 ]
|
||||
do
|
||||
if [ $copy -eq $ROTATE_COPIES ]
|
||||
then
|
||||
rm -rf "$1/$backup.obackup.$copy"
|
||||
fi
|
||||
path="$1/$backup.obackup.$(($copy-1))"
|
||||
if [[ -f $path || -d $path ]]
|
||||
then
|
||||
mv $path "$1/$backup.obackup.$copy"
|
||||
fi
|
||||
copy=$(($copy-1))
|
||||
done
|
||||
|
||||
# Latest file backup will not be moved if script configured for remote backup so next rsync execution will only do delta copy instead of full one
|
||||
if [[ $backup == *.sql.* ]]
|
||||
then
|
||||
mv "$1/$backup" "$1/$backup.obackup.1"
|
||||
elif [ "$REMOTE_BACKUP" == "yes" ]
|
||||
then
|
||||
cp -R "$1/$backup" "$1/$backup.obackup.1"
|
||||
else
|
||||
mv "$1/backup" "$1/$backup.obackup.1"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function Init
|
||||
{
|
||||
# Set error exit code if a piped command fails
|
||||
set -o pipefail
|
||||
set -o errtrace
|
||||
|
||||
trap TrapStop SIGINT SIGQUIT
|
||||
if [ "$DEBUG" == "yes" ]
|
||||
then
|
||||
trap 'TrapError ${LINENO} $?' ERR
|
||||
fi
|
||||
}
|
||||
|
||||
function DryRun
|
||||
{
|
||||
Log "/!\ DRY RUN as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)"
|
||||
|
||||
SetCompressionOptions
|
||||
SetSudoOptions
|
||||
|
||||
if [ "$BACKUP_SQL" != "no" ]
|
||||
then
|
||||
ListDatabases
|
||||
fi
|
||||
if [ "$BACKUP_FILES" != "no" ]
|
||||
then
|
||||
ListDirectories
|
||||
GetDirectoriesSize
|
||||
fi
|
||||
echo "DB backup list: $DATABASES_TO_BACKUP"
|
||||
echo "DB exclude list: $DATABASES_EXCLUDED_LIST"
|
||||
echo "Dirs backup list: $DIRECTORIES_TO_BACKUP"
|
||||
echo "Dirs exclude list: $DIRECTORIES_EXCLUDED_LIST"
|
||||
|
||||
CheckLocalSpace
|
||||
}
|
||||
|
||||
function Main
|
||||
{
|
||||
Log "Backup launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)"
|
||||
|
||||
SetCompressionOptions
|
||||
SetSudoOptions
|
||||
|
||||
if [ "$BACKUP_SQL" != "no" ]
|
||||
then
|
||||
ListDatabases
|
||||
fi
|
||||
if [ "$BACKUP_FILES" != "no" ]
|
||||
then
|
||||
ListDirectories
|
||||
GetDirectoriesSize
|
||||
fi
|
||||
CreateLocalStorageDirectories
|
||||
CheckLocalSpace
|
||||
|
||||
# Make Backup
|
||||
if [ "$BACKUP_SQL" != "no" ]
|
||||
then
|
||||
if [ "$ROTATE_BACKUPS" == "yes" ]
|
||||
then
|
||||
RotateBackups $LOCAL_SQL_STORAGE
|
||||
fi
|
||||
BackupDatabases
|
||||
fi
|
||||
if [ "$BACKUP_FILES" != "no" ]
|
||||
then
|
||||
if [ "$ROTATE_BACKUPS" == "yes" ]
|
||||
then
|
||||
RotateBackups $LOCAL_FILE_STORAGE
|
||||
fi
|
||||
RsyncExcludePattern
|
||||
FilesBackup
|
||||
fi
|
||||
# Be a happy sysadmin (and drink a coffee ? Nahh... it's past midnight.)
|
||||
}
|
||||
|
||||
CheckEnvironment
|
||||
if [ $? == 0 ]
|
||||
then
|
||||
if [ "$1" != "" ]
|
||||
then
|
||||
LoadConfigFile $1
|
||||
if [ $? == 0 ]
|
||||
then
|
||||
Init
|
||||
DATE=$(date)
|
||||
Log "--------------------------------------------------------------------"
|
||||
Log "$DATE - Obackup v$OBACKUP_VERSION script begin."
|
||||
Log "--------------------------------------------------------------------"
|
||||
if [ "$2" == "--dry" ]
|
||||
then
|
||||
DryRun
|
||||
else
|
||||
/root/zsnap_vds2268.sh create
|
||||
Main
|
||||
fi
|
||||
CleanUp
|
||||
else
|
||||
LogError "Configuration file could not be loaded."
|
||||
exit
|
||||
fi
|
||||
else
|
||||
LogError "No configuration file provided."
|
||||
exit
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $error_alert -ne 0 ]
|
||||
then
|
||||
SendAlert
|
||||
LogError "Backup script finished with errors."
|
||||
else
|
||||
Log "Backup script finshed."
|
||||
fi
|
||||
Loading…
Reference in New Issue