Linux
/
obackup
mirror of https://github.com/deajan/obackup.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Rebuilt targets

This commit is contained in:
deajan 2016-08-17 10:01:55 +02:00
parent a84a8c95f9
commit 87dd096f61
3 changed files with 93 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
dev/debug_obackup.sh
View File

@ -1,13 +1,17 @@
#!/usr/bin/env bash
#TODO: test bad return of _GetDirectoriesSizeLocal & Remote
#TODO: test bad return of _GetDirectoriesSizeRemote
#TODO(critical): fix double path in rotate functions (switching from ls to find)
#TODO(high): check paths with spaces (again)
#TODO(low): investigate all exit codes and adapt depending on WARN / ERROR so obackup-batch won't rerun WARN runs
#TODO(low): obackup-rerun is minimal 1 and not 0
###### Remote push/pull (or local) backup script for files & databases
PROGRAM="obackup"
AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
PROGRAM_VERSION=2.1-dev
PROGRAM_BUILD=2016081603
PROGRAM_BUILD=2016081702
IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN ####
@ -1344,7 +1348,7 @@ CAN_BACKUP_FILES=1
function TrapStop {
Logger "/!\ Manual exit of backup script. Backups may be in inconsistent state." "WARN"
exit 1
exit 2
}
function TrapQuit {
@ -1440,7 +1444,7 @@ function CheckCurrentConfig {
fi
# Check all variables that should contain a numerical value >= 0
declare -a num_vars=(BACKUP_SIZE_MINIMUM SQL_WARN_MIN_SPACE FILE_WARN_MIN_SPACE SOFT_MAX_EXEC_TIME_DB_TASK HARD_MAX_EXEC_TIME_DB_TASK COMPRESSION_LEVEL SOFT_MAX_EXEC_TIME_FILE_TASK HARD_MAX_EXEC_TIME_FILE_TASK BANDWIDTH SOFT_MAX_EXEC_TIME_TOTAL HARD_MAX_EXEC_TIME_TOTAL ROTATE_SQL_COPIES ROTATE_FILE_COPIES MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
declare -a num_vars=(BACKUP_SIZE_MINIMUM SQL_WARN_MIN_SPACE FILE_WARN_MIN_SPACE SOFT_MAX_EXEC_TIME_DB_TASK HARD_MAX_EXEC_TIME_DB_TASK COMPRESSION_LEVEL SOFT_MAX_EXEC_TIME_FILE_TASK HARD_MAX_EXEC_TIME_FILE_TASK BANDWIDTH SOFT_MAX_EXEC_TIME_TOTAL HARD_MAX_EXEC_TIME_TOTAL ROTATE_SQL_COPIES ROTATE_FILE_COPIES KEEP_LOGGING MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
for i in "${num_vars[@]}"; do
test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it with the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
@ -2289,22 +2293,22 @@ function _RotateBackupsLocal {
copy=$rotate_copies
while [ $copy -gt 1 ]; do
if [ $copy -eq $rotate_copies ]; then
cmd="rm -rf \"$backup_path/$backup.$PROGRAM.$copy\""
cmd="rm -rf \"$backup.$PROGRAM.$copy\""
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING
if [ $? != 0 ]; then
Logger "Cannot delete oldest copy [$backup_path/$backup.$PROGRAM.$copy]." "ERROR"
Logger "Cannot delete oldest copy [$backup.$PROGRAM.$copy]." "ERROR"
fi
fi
path="$backup_path/$backup.$PROGRAM.$(($copy-1))"
path="$backup.$PROGRAM.$(($copy-1))"
if [[ -f $path || -d $path ]]; then
cmd="mv \"$path\" \"$backup_path/$backup.$PROGRAM.$copy\""
cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\""
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING
if [ $? != 0 ]; then
Logger "Cannot move [$path] to [$backup_path/$backup.$PROGRAM.$copy]." "ERROR"
Logger "Cannot move [$path] to [$backup.$PROGRAM.$copy]." "ERROR"
fi
fi
@ -2313,30 +2317,30 @@ function _RotateBackupsLocal {
# Latest file backup will not be moved if script configured for remote backup so next rsync execution will only do delta copy instead of full one
if [[ $backup == *.sql.* ]]; then
cmd="mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\""
cmd="mv \"$backup\" \"$backup.$PROGRAM.1\""
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING
if [ $? != 0 ]; then
Logger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR"
Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi
elif [ "$REMOTE_OPERATION" == "yes" ]; then
cmd="cp -R \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\""
cmd="cp -R \"$backup\" \"$backup.$PROGRAM.1\""
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING
if [ $? != 0 ]; then
Logger "Cannot copy [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR"
Logger "Cannot copy [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi
else
cmd="mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\""
cmd="mv \"$backup\" \"$backup.$PROGRAM.1\""
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING
if [ $? != 0 ]; then
Logger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR"
Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi
fi
done
@ -2393,20 +2397,20 @@ function _RotateBackupsRemoteSSH {
copy=$rotate_copies
while [ $copy -gt 1 ]; do
if [ $copy -eq $rotate_copies ]; then
cmd="$COMMAND_SUDO rm -rf \"$backup_path/$backup.$PROGRAM.$copy\""
cmd="$COMMAND_SUDO rm -rf \"$backup.$PROGRAM.$copy\""
RemoteLogger "cmd: $cmd" "DEBUG"
eval "$cmd"
if [ $? != 0 ]; then
RemoteLogger "Cannot delete oldest copy [$backup_path/$backup.$PROGRAM.$copy]." "ERROR"
RemoteLogger "Cannot delete oldest copy [$backup.$PROGRAM.$copy]." "ERROR"
fi
fi
path="$backup_path/$backup.$PROGRAM.$(($copy-1))"
path="$backup.$PROGRAM.$(($copy-1))"
if [[ -f $path || -d $path ]]; then
cmd="$COMMAND_SUDO mv \"$path\" \"$backup_path/$backup.$PROGRAM.$copy\""
cmd="$COMMAND_SUDO mv \"$path\" \"$backup.$PROGRAM.$copy\""
RemoteLogger "cmd: $cmd" "DEBUG"
eval "$cmd"
if [ $? != 0 ]; then
RemoteLogger "Cannot move [$path] to [$backup_path/$backup.$PROGRAM.$copy]." "ERROR"
RemoteLogger "Cannot move [$path] to [$backup.$PROGRAM.$copy]." "ERROR"
fi
fi
@ -2415,27 +2419,27 @@ function _RotateBackupsRemoteSSH {
# Latest file backup will not be moved if script configured for remote backup so next rsync execution will only do delta copy instead of full one
if [[ $backup == *.sql.* ]]; then
cmd="$COMMAND_SUDO mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\""
cmd="$COMMAND_SUDO mv \"$backup\" \"$backup.$PROGRAM.1\""
RemoteLogger "cmd: $cmd" "DEBUG"
eval "$cmd"
if [ $? != 0 ]; then
RemoteLogger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR"
RemoteLogger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi
elif [ "$REMOTE_OPERATION" == "yes" ]; then
cmd="$COMMAND_SUDO cp -R \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\""
cmd="$COMMAND_SUDO cp -R \"$backup\" \"$backup.$PROGRAM.1\""
RemoteLogger "cmd: $cmd" "DEBUG"
eval "$cmd"
if [ $? != 0 ]; then
RemoteLogger "Cannot copy [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR"
RemoteLogger "Cannot copy [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi
else
cmd="$COMMAND_SUDO mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\""
cmd="$COMMAND_SUDO mv \"$backup\" \"$backup.$PROGRAM.1\""
RemoteLogger "cmd: $cmd" "DEBUG"
eval "$cmd"
if [ $? != 0 ]; then
RemoteLogger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR"
RemoteLogger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi
fi
done

55
obackup-batch.sh
View File

@ -3,7 +3,7 @@ SUBPROGRAM=obackup
PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones
AUTHOR="(L) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr - ozy@netpower.fr"
PROGRAM_BUILD=2016052501
PROGRAM_BUILD=2016081701
## Runs an osync /obackup instance for every conf file found
## If an instance fails, run it again if time permits
@ -14,8 +14,8 @@ CONF_FILE_PATH=/etc/$SUBPROGRAM
## If maximum execution time is not reached, failed instances will be rerun. Max exec time is in seconds. Example is set to 10 hours.
MAX_EXECUTION_TIME=36000
## Specifies the number of reruns an instance may get
MAX_RERUNS=3
## Specifies the number of total runs an instance may get
MAX_RUNS=3
## Log file path
if [ -w /var/log ]; then
@ -78,38 +78,49 @@ function CheckEnvironment {
}
function Batch {
local runs=0 # Number of batch runs
local runList # Actual conf file list to run
local runAgainList # List of failed conf files sto run again
local confFile
local result
## Get list of .conf files
for i in $CONF_FILE_PATH/*.conf
for confFile in "$CONF_FILE_PATH/*.conf"
do
if [ "$RUN" == "" ]; then
RUN="$i"
if [ "$runList" == "" ]; then
runList="$confFile"
else
RUN=$RUN" $i"
runList=$runList" $confFile"
fi
done
RERUNS=0
while ([ $MAX_EXECUTION_TIME -gt $SECONDS ] || [ $MAX_EXECUTION_TIME -eq 0 ]) && [ "$RUN" != "" ] && [ $MAX_RERUNS -gt $RERUNS ]
while ([ $MAX_EXECUTION_TIME -gt $SECONDS ] || [ $MAX_EXECUTION_TIME -eq 0 ]) && [ "$runList" != "" ] && [ $MAX_RUNS -gt $runs ]
do
Logger "$SUBPROGRAM instances will be run for: $RUN" "NOTICE"
for i in $RUN
Logger "$SUBPROGRAM instances will be run for: $runList" "NOTICE"
for confFile in $runList
do
$SUBPROGRAM_EXECUTABLE "$i" $opts &
$SUBPROGRAM_EXECUTABLE "$confFile" $opts &
wait $!
if [ $? != 0 ]; then
Logger "Run instance $(basename $i) failed" "ERROR"
if [ "$RUN_AGAIN" == "" ]; then
RUN_AGAIN="$i"
else
RUN_AGAIN=$RUN_AGAIN" $i"
result=$?
if [ $result != 0 ]; then
if [ $result == 1 ] || [ $result == 120 ] || [ $result == 128 ]; then
Logger "Run instance $(basename $confFile) failed with exit code [$result]." "ERROR"
if [ "$runAgainList" == "" ]; then
runAgainList="$confFile"
else
runAgainList=$runAgainList" $confFile"
fi
elif [ $result == 2 ]; then
Logger "Run instance $(basename $confFile) finished with warnings." "WARN"
fi
else
Logger "Run instance $(basename $i) succeed." "NOTICE"
Logger "Run instance $(basename $confFile) succeed." "NOTICE"
fi
done
RUN="$RUN_AGAIN"
RUN_AGAIN=""
RERUNS=$(($RERUNS + 1))
runList="$runAgainList"
runAgainList=""
runs=$(($runs + 1))
done
}

56
obackup.sh
View File

@ -1,13 +1,17 @@
#!/usr/bin/env bash
#TODO: test bad return of _GetDirectoriesSizeLocal & Remote
#TODO: test bad return of _GetDirectoriesSizeRemote
#TODO(critical): fix double path in rotate functions (switching from ls to find)
#TODO(high): check paths with spaces (again)
#TODO(low): investigate all exit codes and adapt depending on WARN / ERROR so obackup-batch won't rerun WARN runs
#TODO(low): obackup-rerun is minimal 1 and not 0
###### Remote push/pull (or local) backup script for files & databases
PROGRAM="obackup"
AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
PROGRAM_VERSION=2.1-dev
PROGRAM_BUILD=2016081603
PROGRAM_BUILD=2016081702
IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN ####
@ -1270,7 +1274,7 @@ CAN_BACKUP_FILES=1
function TrapStop {
Logger "/!\ Manual exit of backup script. Backups may be in inconsistent state." "WARN"
exit 1
exit 2
}
function TrapQuit {
@ -1364,7 +1368,7 @@ function CheckCurrentConfig {
fi
# Check all variables that should contain a numerical value >= 0
declare -a num_vars=(BACKUP_SIZE_MINIMUM SQL_WARN_MIN_SPACE FILE_WARN_MIN_SPACE SOFT_MAX_EXEC_TIME_DB_TASK HARD_MAX_EXEC_TIME_DB_TASK COMPRESSION_LEVEL SOFT_MAX_EXEC_TIME_FILE_TASK HARD_MAX_EXEC_TIME_FILE_TASK BANDWIDTH SOFT_MAX_EXEC_TIME_TOTAL HARD_MAX_EXEC_TIME_TOTAL ROTATE_SQL_COPIES ROTATE_FILE_COPIES MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
declare -a num_vars=(BACKUP_SIZE_MINIMUM SQL_WARN_MIN_SPACE FILE_WARN_MIN_SPACE SOFT_MAX_EXEC_TIME_DB_TASK HARD_MAX_EXEC_TIME_DB_TASK COMPRESSION_LEVEL SOFT_MAX_EXEC_TIME_FILE_TASK HARD_MAX_EXEC_TIME_FILE_TASK BANDWIDTH SOFT_MAX_EXEC_TIME_TOTAL HARD_MAX_EXEC_TIME_TOTAL ROTATE_SQL_COPIES ROTATE_FILE_COPIES KEEP_LOGGING MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
for i in "${num_vars[@]}"; do
test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it with the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
@ -2187,22 +2191,22 @@ function _RotateBackupsLocal {
copy=$rotate_copies
while [ $copy -gt 1 ]; do
if [ $copy -eq $rotate_copies ]; then
cmd="rm -rf \"$backup_path/$backup.$PROGRAM.$copy\""
cmd="rm -rf \"$backup.$PROGRAM.$copy\""
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING
if [ $? != 0 ]; then
Logger "Cannot delete oldest copy [$backup_path/$backup.$PROGRAM.$copy]." "ERROR"
Logger "Cannot delete oldest copy [$backup.$PROGRAM.$copy]." "ERROR"
fi
fi
path="$backup_path/$backup.$PROGRAM.$(($copy-1))"
path="$backup.$PROGRAM.$(($copy-1))"
if [[ -f $path || -d $path ]]; then
cmd="mv \"$path\" \"$backup_path/$backup.$PROGRAM.$copy\""
cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\""
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING
if [ $? != 0 ]; then
Logger "Cannot move [$path] to [$backup_path/$backup.$PROGRAM.$copy]." "ERROR"
Logger "Cannot move [$path] to [$backup.$PROGRAM.$copy]." "ERROR"
fi
fi
@ -2211,30 +2215,30 @@ function _RotateBackupsLocal {
# Latest file backup will not be moved if script configured for remote backup so next rsync execution will only do delta copy instead of full one
if [[ $backup == *.sql.* ]]; then
cmd="mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\""
cmd="mv \"$backup\" \"$backup.$PROGRAM.1\""
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING
if [ $? != 0 ]; then
Logger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR"
Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi
elif [ "$REMOTE_OPERATION" == "yes" ]; then
cmd="cp -R \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\""
cmd="cp -R \"$backup\" \"$backup.$PROGRAM.1\""
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING
if [ $? != 0 ]; then
Logger "Cannot copy [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR"
Logger "Cannot copy [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi
else
cmd="mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\""
cmd="mv \"$backup\" \"$backup.$PROGRAM.1\""
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING
if [ $? != 0 ]; then
Logger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR"
Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi
fi
done
@ -2285,20 +2289,20 @@ function _RotateBackupsRemoteSSH {
copy=$rotate_copies
while [ $copy -gt 1 ]; do
if [ $copy -eq $rotate_copies ]; then
cmd="$COMMAND_SUDO rm -rf \"$backup_path/$backup.$PROGRAM.$copy\""
cmd="$COMMAND_SUDO rm -rf \"$backup.$PROGRAM.$copy\""
RemoteLogger "cmd: $cmd" "DEBUG"
eval "$cmd"
if [ $? != 0 ]; then
RemoteLogger "Cannot delete oldest copy [$backup_path/$backup.$PROGRAM.$copy]." "ERROR"
RemoteLogger "Cannot delete oldest copy [$backup.$PROGRAM.$copy]." "ERROR"
fi
fi
path="$backup_path/$backup.$PROGRAM.$(($copy-1))"
path="$backup.$PROGRAM.$(($copy-1))"
if [[ -f $path || -d $path ]]; then
cmd="$COMMAND_SUDO mv \"$path\" \"$backup_path/$backup.$PROGRAM.$copy\""
cmd="$COMMAND_SUDO mv \"$path\" \"$backup.$PROGRAM.$copy\""
RemoteLogger "cmd: $cmd" "DEBUG"
eval "$cmd"
if [ $? != 0 ]; then
RemoteLogger "Cannot move [$path] to [$backup_path/$backup.$PROGRAM.$copy]." "ERROR"
RemoteLogger "Cannot move [$path] to [$backup.$PROGRAM.$copy]." "ERROR"
fi
fi
@ -2307,27 +2311,27 @@ function _RotateBackupsRemoteSSH {
# Latest file backup will not be moved if script configured for remote backup so next rsync execution will only do delta copy instead of full one
if [[ $backup == *.sql.* ]]; then
cmd="$COMMAND_SUDO mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\""
cmd="$COMMAND_SUDO mv \"$backup\" \"$backup.$PROGRAM.1\""
RemoteLogger "cmd: $cmd" "DEBUG"
eval "$cmd"
if [ $? != 0 ]; then
RemoteLogger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR"
RemoteLogger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi
elif [ "$REMOTE_OPERATION" == "yes" ]; then
cmd="$COMMAND_SUDO cp -R \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\""
cmd="$COMMAND_SUDO cp -R \"$backup\" \"$backup.$PROGRAM.1\""
RemoteLogger "cmd: $cmd" "DEBUG"
eval "$cmd"
if [ $? != 0 ]; then
RemoteLogger "Cannot copy [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR"
RemoteLogger "Cannot copy [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi
else
cmd="$COMMAND_SUDO mv \"$backup_path/$backup\" \"$backup_path/$backup.$PROGRAM.1\""
cmd="$COMMAND_SUDO mv \"$backup\" \"$backup.$PROGRAM.1\""
RemoteLogger "cmd: $cmd" "DEBUG"
eval "$cmd"
if [ $? != 0 ]; then
RemoteLogger "Cannot move [$backup_path/$backup] to [$backup_path/$backup.$PROGRAM.1]." "ERROR"
RemoteLogger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi
fi
done