Linux/obackup
1
0
Fork 0
mirror of https://github.com/deajan/obackup.git synced 2024-12-25 23:13:41 +01:00
Code Issues Releases Wiki Activity

Rebuilt targets

Browse Source
This commit is contained in:
deajan 2016-08-18 11:53:18 +02:00
parent 9fed22d4f9
commit b7c49ed6d9
3 changed files with 115 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
dev/debug_obackup.sh
View File

@ -1,22 +1,21 @@
#!/usr/bin/env bash #!/usr/bin/env bash
#TODO: test bad return of _GetDirectoriesSizeRemote #TODO: test bad return of _GetDirectoriesSizeRemote
#TODO(critical): fix double path in rotate functions (switching from ls to find) #TODO(critical): test RotateCopies remote
#TODO(high): check paths with spaces (again) #TODO(high): check paths with spaces (again) - fails
#TODO(low): investigate all exit codes and adapt depending on WARN / ERROR so obackup-batch won't rerun WARN runs #TODO(low): doc obackup-rerun is minimal 1 and not 0
#TODO(low): obackup-rerun is minimal 1 and not 0
###### Remote push/pull (or local) backup script for files & databases ###### Remote push/pull (or local) backup script for files & databases
PROGRAM="obackup" PROGRAM="obackup"
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
PROGRAM_VERSION=2.1-dev PROGRAM_VERSION=2.1-dev
PROGRAM_BUILD=2016081702 PROGRAM_BUILD=2016081803
IS_STABLE=no IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN #### #### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016081605 ## FUNC_BUILD=2016081701
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -620,7 +619,7 @@ function WaitForTaskCompletion {
if kill -0 $pid > /dev/null 2>&1; then if kill -0 $pid > /dev/null 2>&1; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array # Handle uninterruptible sleep state or zombies by ommiting them from running process array
#TODO(high): have this tested on *BSD, Mac & Win #TODO(high): have this tested on *BSD, Mac & Win
pidState=$(ps -p$pid -o state=) pidState=$(ps -p$pid -o state= 2 > /dev/null)
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
newPidsArray+=($pid) newPidsArray+=($pid)
fi fi
@ -734,7 +733,7 @@ function EscapeSpaces {
} }
function IsNumeric { function IsNumeric {
eval "local value=\"${1}\"" # Needed so variable variables can be processed eval "local value=\"${1}\"" # Needed eval so variable variables can be processed
local re="^-?[0-9]+([.][0-9]+)?$" local re="^-?[0-9]+([.][0-9]+)?$"
if [[ $value =~ $re ]]; then if [[ $value =~ $re ]]; then
@ -763,9 +762,9 @@ function urlEncode {
} }
function urlDecode { function urlDecode {
local url_encoded="${1//+/ }" local url_encoded="${1//+/ }"
printf '%b' "${url_encoded//%/\\x}" printf '%b' "${url_encoded//%/\\x}"
} }
function GetLocalOS { function GetLocalOS {
@ -1341,7 +1340,8 @@ PARTIAL_DIR=".obackup_workdir_partial"
# $FILE_BACKUP_TASKS list of directories to backup, found in config file # $FILE_BACKUP_TASKS list of directories to backup, found in config file
# $FILE_RECURSIVE_BACKUP_TASKS, list of directories to backup, computed from config file recursive list # $FILE_RECURSIVE_BACKUP_TASKS, list of directories to backup, computed from config file recursive list
# $FILE_RECURSIVE_EXCLUDED_TASKS, list of all directories excluded from recursive list # $FILE_RECURSIVE_EXCLUDED_TASKS, list of all directories excluded from recursive list
# $FILE_SIZE_LIST, list of all directories to include in GetDirectoriesSize # $FILE_SIZE_LIST_LOCAL, list of all directories to include in GetDirectoriesSize, enclosed by escaped doublequotes for local command
# $FILE_SIZE_LIST_LOCAL, list of all directories to include in GetDirectoriesSize, enclosed by escaped singlequotes for remote command
CAN_BACKUP_SQL=1 CAN_BACKUP_SQL=1
CAN_BACKUP_FILES=1 CAN_BACKUP_FILES=1
@ -1560,7 +1560,7 @@ function ListDatabases {
if [ "$DATABASES_ALL" == "yes" ]; then if [ "$DATABASES_ALL" == "yes" ]; then
dbBackup=1 dbBackup=1
IFs=$PATH_SEPARATOR_CHAR read -r -a dbArray <<< "$DATABASES_ALL_EXCLUDE_LIST" IFS=$PATH_SEPARATOR_CHAR read -r -a dbArray <<< "$DATABASES_ALL_EXCLUDE_LIST"
for j in "${dbArray[@]}"; do for j in "${dbArray[@]}"; do
if [ "$dbName" == "$j" ]; then if [ "$dbName" == "$j" ]; then
dbBackup=0 dbBackup=0
@ -1568,7 +1568,7 @@ function ListDatabases {
done done
else else
dbBackup=0 dbBackup=0
IFs=$PATH_SEPARATOR_CHAR read -r -a dbArray <<< "$DATABASES_LIST" IFS=$PATH_SEPARATOR_CHAR read -r -a dbArray <<< "$DATABASES_LIST"
for j in "${dbArray[@]}"; do for j in "${dbArray[@]}"; do
if [ "$dbName" == "$j" ]; then if [ "$dbName" == "$j" ]; then
dbBackup=1 dbBackup=1
@ -1694,11 +1694,13 @@ function ListRecursiveBackupDirectories {
if [ $file_exclude -eq 0 ]; then if [ $file_exclude -eq 0 ]; then
if [ "$FILE_RECURSIVE_BACKUP_TASKS" == "" ]; then if [ "$FILE_RECURSIVE_BACKUP_TASKS" == "" ]; then
FILE_SIZE_LIST_LOCAL="\"$line\""
FILE_SIZE_LIST_REMOTE="\'$line\'"
FILE_RECURSIVE_BACKUP_TASKS="$line" FILE_RECURSIVE_BACKUP_TASKS="$line"
FILE_SIZE_LIST="$(EscapeSpaces $line)"
else else
FILE_SIZE_LIST_LOCAL="$FILE_SIZE_LIST_LOCAL \"$line\""
FILE_SIZE_LIST_REMOTE="$FILE_SIZE_LIST_REMOTE \'$line\'"
FILE_RECURSIVE_BACKUP_TASKS="$FILE_RECURSIVE_BACKUP_TASKS$PATH_SEPARATOR_CHAR$line" FILE_RECURSIVE_BACKUP_TASKS="$FILE_RECURSIVE_BACKUP_TASKS$PATH_SEPARATOR_CHAR$line"
FILE_SIZE_LIST="$FILE_SIZE_LIST $(EscapeSpaces $line)"
fi fi
else else
FILE_RECURSIVE_EXCLUDED_TASKS="$FILE_RECURSIVE_EXCLUDED_TASKS$PATH_SEPARATOR_CHAR$line" FILE_RECURSIVE_EXCLUDED_TASKS="$FILE_RECURSIVE_EXCLUDED_TASKS$PATH_SEPARATOR_CHAR$line"
@ -1708,7 +1710,14 @@ function ListRecursiveBackupDirectories {
IFS=$PATH_SEPARATOR_CHAR read -r -a fileArray <<< "$DIRECTORY_LIST" IFS=$PATH_SEPARATOR_CHAR read -r -a fileArray <<< "$DIRECTORY_LIST"
for directory in "${fileArray[@]}"; do for directory in "${fileArray[@]}"; do
FILE_SIZE_LIST="$FILE_SIZE_LIST $(EscapeSpaces $directory)" if [ "$FILE_SIZE_LIST_LOCAL" == "" ]; then
FILE_SIZE_LIST_LOCAL="\"$directory\""
FILE_SIZE_LIST_REMOTE="\'$directory\'"
else
FILE_SIZE_LIST_LOCAL="$FILE_SIZE_LIST_LOCAL \"$directory\""
FILE_SIZE_LIST_REMOTE="$FILE_SIZE_LIST_REMOTE \'$directory\'"
fi
if [ "$FILE_BACKUP_TASKS" == "" ]; then if [ "$FILE_BACKUP_TASKS" == "" ]; then
FILE_BACKUP_TASKS="$directory" FILE_BACKUP_TASKS="$directory"
else else
@ -1724,7 +1733,8 @@ function _GetDirectoriesSizeLocal {
local cmd local cmd
# No sudo here, assuming you should have all the necessary rights # No sudo here, assuming you should have all the necessary rights
cmd='echo "'$dir_list'" | xargs du -cs | tail -n1 | cut -f1 > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID # This is not pretty, but works with all supported systems
cmd="du -cs $dir_list | tail -n1 | cut -f1 > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID"
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING
@ -1755,7 +1765,7 @@ function _GetDirectoriesSizeRemote {
local cmd local cmd
# Error output is different from stdout because not all files in list may fail at once # Error output is different from stdout because not all files in list may fail at once
cmd=$SSH_CMD' "echo '$dir_list' | xargs '$COMMAND_SUDO' du -cs | tail -n1 | cut -f1" > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID cmd=$SSH_CMD' '$COMMAND_SUDO' du -cs '$dir_list' | tail -n1 | cut -f1 > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING
@ -1785,11 +1795,11 @@ function GetDirectoriesSize {
if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]; then if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]; then
if [ "$FILE_BACKUP" != "no" ]; then if [ "$FILE_BACKUP" != "no" ]; then
_GetDirectoriesSizeLocal "$FILE_SIZE_LIST" _GetDirectoriesSizeLocal "$FILE_SIZE_LIST_LOCAL"
fi fi
elif [ "$BACKUP_TYPE" == "pull" ]; then elif [ "$BACKUP_TYPE" == "pull" ]; then
if [ "$FILE_BACKUP" != "no" ]; then if [ "$FILE_BACKUP" != "no" ]; then
_GetDirectoriesSizeRemote "$FILE_SIZE_LIST" _GetDirectoriesSizeRemote "$FILE_SIZE_LIST_REMOTE"
fi fi
fi fi
} }
@ -2289,20 +2299,25 @@ function _RotateBackupsLocal {
local cmd local cmd
local path local path
find "$backup_path" -iname "*.$PROGRAM.*" -print0 | while IFS= read -r -d $'\0' backup; do #TODO(low): check if mindepth / maxdepth is compatible BSD & MSYS
find "$backup_path" -mindepth 1 -maxdepth 1 ! -iname "*.$PROGRAM.*" -print0 | while IFS= read -r -d $'\0' backup; do
copy=$rotate_copies copy=$rotate_copies
while [ $copy -gt 1 ]; do while [ $copy -gt 1 ]; do
if [ $copy -eq $rotate_copies ]; then if [ $copy -eq $rotate_copies ]; then
cmd="rm -rf \"$backup.$PROGRAM.$copy\"" path="$backup.$PROGRAM.$copy"
Logger "cmd: $cmd" "DEBUG" if [ -f "$path" ] || [ -d "$path" ]; then
eval "$cmd" & cmd="rm -rf \"$path\""
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING Logger "cmd: $cmd" "DEBUG"
if [ $? != 0 ]; then eval "$cmd" &
Logger "Cannot delete oldest copy [$backup.$PROGRAM.$copy]." "ERROR" WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING
if [ $? != 0 ]; then
Logger "Cannot delete oldest copy [$path]." "ERROR"
fi
fi fi
fi fi
path="$backup.$PROGRAM.$(($copy-1))" path="$backup.$PROGRAM.$(($copy-1))"
if [[ -f $path || -d $path ]]; then if [ -f "$path" ] || [ -d "$path" ]; then
cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\"" cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\""
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
@ -2393,19 +2408,22 @@ function RemoteLogger {
} }
function _RotateBackupsRemoteSSH { function _RotateBackupsRemoteSSH {
find "$backup_path" -iname "*.$PROGRAM.*" -print0 | while IFS= read -r -d $'\0' backup; do find "$backup_path" -mindepth 1 -maxdepth 1 ! -iname "*.$PROGRAM.*" -print0 | while IFS= read -r -d $'\0' backup; do
copy=$rotate_copies copy=$rotate_copies
while [ $copy -gt 1 ]; do while [ $copy -gt 1 ]; do
if [ $copy -eq $rotate_copies ]; then if [ $copy -eq $rotate_copies ]; then
cmd="$COMMAND_SUDO rm -rf \"$backup.$PROGRAM.$copy\"" path="$backup.$PROGRAM.$copy"
RemoteLogger "cmd: $cmd" "DEBUG" if [ -f "$path" ] || [ -d "$path" ]; then
eval "$cmd" cmd="$COMMAND_SUDO rm -rf \"$path\""
if [ $? != 0 ]; then RemoteLogger "cmd: $cmd" "DEBUG"
RemoteLogger "Cannot delete oldest copy [$backup.$PROGRAM.$copy]." "ERROR" eval "$cmd"
if [ $? != 0 ]; then
RemoteLogger "Cannot delete oldest copy [$path]." "ERROR"
fi
fi fi
fi fi
path="$backup.$PROGRAM.$(($copy-1))" path="$backup.$PROGRAM.$(($copy-1))"
if [[ -f $path || -d $path ]]; then if [ -f "$path" ] || [ -d "$path" ]; then
cmd="$COMMAND_SUDO mv \"$path\" \"$backup.$PROGRAM.$copy\"" cmd="$COMMAND_SUDO mv \"$path\" \"$backup.$PROGRAM.$copy\""
RemoteLogger "cmd: $cmd" "DEBUG" RemoteLogger "cmd: $cmd" "DEBUG"
eval "$cmd" eval "$cmd"
@ -2466,7 +2484,7 @@ function RotateBackups {
local rotate_copies="${2}" local rotate_copies="${2}"
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
Logger "Rotating backups." "NOTICE" Logger "Rotating backups in [$backup_path] for [$rotate_copies] copies." "NOTICE"
if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "pull" ]; then if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "pull" ]; then
_RotateBackupsLocal "$backup_path" "$rotate_copies" _RotateBackupsLocal "$backup_path" "$rotate_copies"

9
obackup-batch.sh
View File

@ -3,11 +3,16 @@ SUBPROGRAM=obackup
PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones
AUTHOR="(L) 2013-2016 by Orsiris de Jong" AUTHOR="(L) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr - ozy@netpower.fr" CONTACT="http://www.netpower.fr - ozy@netpower.fr"
PROGRAM_BUILD=2016081703 PROGRAM_BUILD=2016081704
## Runs an osync /obackup instance for every conf file found ## Runs an osync /obackup instance for every conf file found
## If an instance fails, run it again if time permits ## If an instance fails, run it again if time permits
if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2"
exit 127
fi
## Configuration file path. The path where all the osync / obackup conf files are, usually /etc/osync or /etc/obackup ## Configuration file path. The path where all the osync / obackup conf files are, usually /etc/osync or /etc/obackup
CONF_FILE_PATH=/etc/$SUBPROGRAM CONF_FILE_PATH=/etc/$SUBPROGRAM
@ -110,7 +115,7 @@ function Batch {
wait $! wait $!
result=$? result=$?
if [ $result != 0 ]; then if [ $result != 0 ]; then
if [ $result == 1 ] || [ $result == 120 ] || [ $result == 128 ]; then if [ $result == 1 ] || [ $result == 128 ]; then # Do not handle exit code 127 because it is already handled here
Logger "Run instance $(basename $confFile) failed with exit code [$result]." "ERROR" Logger "Run instance $(basename $confFile) failed with exit code [$result]." "ERROR"
if [ "$runAgainList" == "" ]; then if [ "$runAgainList" == "" ]; then
runAgainList="$confFile" runAgainList="$confFile"

90
obackup.sh
View File

@ -1,22 +1,21 @@
#!/usr/bin/env bash #!/usr/bin/env bash
#TODO: test bad return of _GetDirectoriesSizeRemote #TODO: test bad return of _GetDirectoriesSizeRemote
#TODO(critical): fix double path in rotate functions (switching from ls to find) #TODO(critical): test RotateCopies remote
#TODO(high): check paths with spaces (again) #TODO(high): check paths with spaces (again) - fails
#TODO(low): investigate all exit codes and adapt depending on WARN / ERROR so obackup-batch won't rerun WARN runs #TODO(low): doc obackup-rerun is minimal 1 and not 0
#TODO(low): obackup-rerun is minimal 1 and not 0
###### Remote push/pull (or local) backup script for files & databases ###### Remote push/pull (or local) backup script for files & databases
PROGRAM="obackup" PROGRAM="obackup"
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
PROGRAM_VERSION=2.1-dev PROGRAM_VERSION=2.1-dev
PROGRAM_BUILD=2016081702 PROGRAM_BUILD=2016081803
IS_STABLE=no IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN #### #### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016081605 ## FUNC_BUILD=2016081701
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -601,7 +600,7 @@ function WaitForTaskCompletion {
if kill -0 $pid > /dev/null 2>&1; then if kill -0 $pid > /dev/null 2>&1; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array # Handle uninterruptible sleep state or zombies by ommiting them from running process array
#TODO(high): have this tested on *BSD, Mac & Win #TODO(high): have this tested on *BSD, Mac & Win
pidState=$(ps -p$pid -o state=) pidState=$(ps -p$pid -o state= 2 > /dev/null)
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
newPidsArray+=($pid) newPidsArray+=($pid)
fi fi
@ -713,7 +712,7 @@ function EscapeSpaces {
} }
function IsNumeric { function IsNumeric {
eval "local value=\"${1}\"" # Needed so variable variables can be processed eval "local value=\"${1}\"" # Needed eval so variable variables can be processed
local re="^-?[0-9]+([.][0-9]+)?$" local re="^-?[0-9]+([.][0-9]+)?$"
if [[ $value =~ $re ]]; then if [[ $value =~ $re ]]; then
@ -742,9 +741,9 @@ function urlEncode {
} }
function urlDecode { function urlDecode {
local url_encoded="${1//+/ }" local url_encoded="${1//+/ }"
printf '%b' "${url_encoded//%/\\x}" printf '%b' "${url_encoded//%/\\x}"
} }
function GetLocalOS { function GetLocalOS {
@ -1267,7 +1266,8 @@ PARTIAL_DIR=".obackup_workdir_partial"
# $FILE_BACKUP_TASKS list of directories to backup, found in config file # $FILE_BACKUP_TASKS list of directories to backup, found in config file
# $FILE_RECURSIVE_BACKUP_TASKS, list of directories to backup, computed from config file recursive list # $FILE_RECURSIVE_BACKUP_TASKS, list of directories to backup, computed from config file recursive list
# $FILE_RECURSIVE_EXCLUDED_TASKS, list of all directories excluded from recursive list # $FILE_RECURSIVE_EXCLUDED_TASKS, list of all directories excluded from recursive list
# $FILE_SIZE_LIST, list of all directories to include in GetDirectoriesSize # $FILE_SIZE_LIST_LOCAL, list of all directories to include in GetDirectoriesSize, enclosed by escaped doublequotes for local command
# $FILE_SIZE_LIST_LOCAL, list of all directories to include in GetDirectoriesSize, enclosed by escaped singlequotes for remote command
CAN_BACKUP_SQL=1 CAN_BACKUP_SQL=1
CAN_BACKUP_FILES=1 CAN_BACKUP_FILES=1
@ -1480,7 +1480,7 @@ function ListDatabases {
if [ "$DATABASES_ALL" == "yes" ]; then if [ "$DATABASES_ALL" == "yes" ]; then
dbBackup=1 dbBackup=1
IFs=$PATH_SEPARATOR_CHAR read -r -a dbArray <<< "$DATABASES_ALL_EXCLUDE_LIST" IFS=$PATH_SEPARATOR_CHAR read -r -a dbArray <<< "$DATABASES_ALL_EXCLUDE_LIST"
for j in "${dbArray[@]}"; do for j in "${dbArray[@]}"; do
if [ "$dbName" == "$j" ]; then if [ "$dbName" == "$j" ]; then
dbBackup=0 dbBackup=0
@ -1488,7 +1488,7 @@ function ListDatabases {
done done
else else
dbBackup=0 dbBackup=0
IFs=$PATH_SEPARATOR_CHAR read -r -a dbArray <<< "$DATABASES_LIST" IFS=$PATH_SEPARATOR_CHAR read -r -a dbArray <<< "$DATABASES_LIST"
for j in "${dbArray[@]}"; do for j in "${dbArray[@]}"; do
if [ "$dbName" == "$j" ]; then if [ "$dbName" == "$j" ]; then
dbBackup=1 dbBackup=1
@ -1611,11 +1611,13 @@ function ListRecursiveBackupDirectories {
if [ $file_exclude -eq 0 ]; then if [ $file_exclude -eq 0 ]; then
if [ "$FILE_RECURSIVE_BACKUP_TASKS" == "" ]; then if [ "$FILE_RECURSIVE_BACKUP_TASKS" == "" ]; then
FILE_SIZE_LIST_LOCAL="\"$line\""
FILE_SIZE_LIST_REMOTE="\'$line\'"
FILE_RECURSIVE_BACKUP_TASKS="$line" FILE_RECURSIVE_BACKUP_TASKS="$line"
FILE_SIZE_LIST="$(EscapeSpaces $line)"
else else
FILE_SIZE_LIST_LOCAL="$FILE_SIZE_LIST_LOCAL \"$line\""
FILE_SIZE_LIST_REMOTE="$FILE_SIZE_LIST_REMOTE \'$line\'"
FILE_RECURSIVE_BACKUP_TASKS="$FILE_RECURSIVE_BACKUP_TASKS$PATH_SEPARATOR_CHAR$line" FILE_RECURSIVE_BACKUP_TASKS="$FILE_RECURSIVE_BACKUP_TASKS$PATH_SEPARATOR_CHAR$line"
FILE_SIZE_LIST="$FILE_SIZE_LIST $(EscapeSpaces $line)"
fi fi
else else
FILE_RECURSIVE_EXCLUDED_TASKS="$FILE_RECURSIVE_EXCLUDED_TASKS$PATH_SEPARATOR_CHAR$line" FILE_RECURSIVE_EXCLUDED_TASKS="$FILE_RECURSIVE_EXCLUDED_TASKS$PATH_SEPARATOR_CHAR$line"
@ -1625,7 +1627,14 @@ function ListRecursiveBackupDirectories {
IFS=$PATH_SEPARATOR_CHAR read -r -a fileArray <<< "$DIRECTORY_LIST" IFS=$PATH_SEPARATOR_CHAR read -r -a fileArray <<< "$DIRECTORY_LIST"
for directory in "${fileArray[@]}"; do for directory in "${fileArray[@]}"; do
FILE_SIZE_LIST="$FILE_SIZE_LIST $(EscapeSpaces $directory)" if [ "$FILE_SIZE_LIST_LOCAL" == "" ]; then
FILE_SIZE_LIST_LOCAL="\"$directory\""
FILE_SIZE_LIST_REMOTE="\'$directory\'"
else
FILE_SIZE_LIST_LOCAL="$FILE_SIZE_LIST_LOCAL \"$directory\""
FILE_SIZE_LIST_REMOTE="$FILE_SIZE_LIST_REMOTE \'$directory\'"
fi
if [ "$FILE_BACKUP_TASKS" == "" ]; then if [ "$FILE_BACKUP_TASKS" == "" ]; then
FILE_BACKUP_TASKS="$directory" FILE_BACKUP_TASKS="$directory"
else else
@ -1640,7 +1649,8 @@ function _GetDirectoriesSizeLocal {
local cmd local cmd
# No sudo here, assuming you should have all the necessary rights # No sudo here, assuming you should have all the necessary rights
cmd='echo "'$dir_list'" | xargs du -cs | tail -n1 | cut -f1 > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID # This is not pretty, but works with all supported systems
cmd="du -cs $dir_list | tail -n1 | cut -f1 > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID"
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING
@ -1670,7 +1680,7 @@ function _GetDirectoriesSizeRemote {
local cmd local cmd
# Error output is different from stdout because not all files in list may fail at once # Error output is different from stdout because not all files in list may fail at once
cmd=$SSH_CMD' "echo '$dir_list' | xargs '$COMMAND_SUDO' du -cs | tail -n1 | cut -f1" > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID cmd=$SSH_CMD' '$COMMAND_SUDO' du -cs '$dir_list' | tail -n1 | cut -f1 > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING
@ -1699,11 +1709,11 @@ function GetDirectoriesSize {
if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]; then if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]; then
if [ "$FILE_BACKUP" != "no" ]; then if [ "$FILE_BACKUP" != "no" ]; then
_GetDirectoriesSizeLocal "$FILE_SIZE_LIST" _GetDirectoriesSizeLocal "$FILE_SIZE_LIST_LOCAL"
fi fi
elif [ "$BACKUP_TYPE" == "pull" ]; then elif [ "$BACKUP_TYPE" == "pull" ]; then
if [ "$FILE_BACKUP" != "no" ]; then if [ "$FILE_BACKUP" != "no" ]; then
_GetDirectoriesSizeRemote "$FILE_SIZE_LIST" _GetDirectoriesSizeRemote "$FILE_SIZE_LIST_REMOTE"
fi fi
fi fi
} }
@ -2187,20 +2197,25 @@ function _RotateBackupsLocal {
local cmd local cmd
local path local path
find "$backup_path" -iname "*.$PROGRAM.*" -print0 | while IFS= read -r -d $'\0' backup; do #TODO(low): check if mindepth / maxdepth is compatible BSD & MSYS
find "$backup_path" -mindepth 1 -maxdepth 1 ! -iname "*.$PROGRAM.*" -print0 | while IFS= read -r -d $'\0' backup; do
copy=$rotate_copies copy=$rotate_copies
while [ $copy -gt 1 ]; do while [ $copy -gt 1 ]; do
if [ $copy -eq $rotate_copies ]; then if [ $copy -eq $rotate_copies ]; then
cmd="rm -rf \"$backup.$PROGRAM.$copy\"" path="$backup.$PROGRAM.$copy"
Logger "cmd: $cmd" "DEBUG" if [ -f "$path" ] || [ -d "$path" ]; then
eval "$cmd" & cmd="rm -rf \"$path\""
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING Logger "cmd: $cmd" "DEBUG"
if [ $? != 0 ]; then eval "$cmd" &
Logger "Cannot delete oldest copy [$backup.$PROGRAM.$copy]." "ERROR" WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING
if [ $? != 0 ]; then
Logger "Cannot delete oldest copy [$path]." "ERROR"
fi
fi fi
fi fi
path="$backup.$PROGRAM.$(($copy-1))" path="$backup.$PROGRAM.$(($copy-1))"
if [[ -f $path || -d $path ]]; then if [ -f "$path" ] || [ -d "$path" ]; then
cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\"" cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\""
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
@ -2285,19 +2300,22 @@ function RemoteLogger {
} }
function _RotateBackupsRemoteSSH { function _RotateBackupsRemoteSSH {
find "$backup_path" -iname "*.$PROGRAM.*" -print0 | while IFS= read -r -d $'\0' backup; do find "$backup_path" -mindepth 1 -maxdepth 1 ! -iname "*.$PROGRAM.*" -print0 | while IFS= read -r -d $'\0' backup; do
copy=$rotate_copies copy=$rotate_copies
while [ $copy -gt 1 ]; do while [ $copy -gt 1 ]; do
if [ $copy -eq $rotate_copies ]; then if [ $copy -eq $rotate_copies ]; then
cmd="$COMMAND_SUDO rm -rf \"$backup.$PROGRAM.$copy\"" path="$backup.$PROGRAM.$copy"
RemoteLogger "cmd: $cmd" "DEBUG" if [ -f "$path" ] || [ -d "$path" ]; then
eval "$cmd" cmd="$COMMAND_SUDO rm -rf \"$path\""
if [ $? != 0 ]; then RemoteLogger "cmd: $cmd" "DEBUG"
RemoteLogger "Cannot delete oldest copy [$backup.$PROGRAM.$copy]." "ERROR" eval "$cmd"
if [ $? != 0 ]; then
RemoteLogger "Cannot delete oldest copy [$path]." "ERROR"
fi
fi fi
fi fi
path="$backup.$PROGRAM.$(($copy-1))" path="$backup.$PROGRAM.$(($copy-1))"
if [[ -f $path || -d $path ]]; then if [ -f "$path" ] || [ -d "$path" ]; then
cmd="$COMMAND_SUDO mv \"$path\" \"$backup.$PROGRAM.$copy\"" cmd="$COMMAND_SUDO mv \"$path\" \"$backup.$PROGRAM.$copy\""
RemoteLogger "cmd: $cmd" "DEBUG" RemoteLogger "cmd: $cmd" "DEBUG"
eval "$cmd" eval "$cmd"
@ -2357,7 +2375,7 @@ function RotateBackups {
local backup_path="${1}" local backup_path="${1}"
local rotate_copies="${2}" local rotate_copies="${2}"
Logger "Rotating backups." "NOTICE" Logger "Rotating backups in [$backup_path] for [$rotate_copies] copies." "NOTICE"
if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "pull" ]; then if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "pull" ]; then
_RotateBackupsLocal "$backup_path" "$rotate_copies" _RotateBackupsLocal "$backup_path" "$rotate_copies"