Linux/obackup
1
0
Fork 0
mirror of https://github.com/deajan/obackup.git synced 2025-01-12 07:03:54 +01:00
Code Issues Releases Wiki Activity

Update file backup paths when encryption is used

Browse Source
This commit is contained in:
deajan 2016-12-28 22:18:59 +01:00
parent a8b6bcbf13
commit 805239e669
1 changed files with 72 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
dev/n_obackup.sh
View File

@ -10,7 +10,7 @@ PROGRAM="obackup"
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
PROGRAM_VERSION=2.1-dev PROGRAM_VERSION=2.1-dev
PROGRAM_BUILD=2016122701 PROGRAM_BUILD=2016122801
IS_STABLE=no IS_STABLE=no
include #### OFUNCTIONS FULL SUBSET #### include #### OFUNCTIONS FULL SUBSET ####
@ -544,7 +544,6 @@ function _GetDirectoriesSizeRemote {
local cmd local cmd
#TODO(med): check if heredoc needed for compat
# Error output is different from stdout because not all files in list may fail at once # Error output is different from stdout because not all files in list may fail at once
$SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ $SSH_CMD env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \
env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \ env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" \
@ -1085,6 +1084,14 @@ function EncryptFiles {
local cryptFileExtension="$CRYPT_FILE_EXTENSION" local cryptFileExtension="$CRYPT_FILE_EXTENSION"
local recursiveArgs="" local recursiveArgs=""
if [ ! -d "$destPath" ]; then
mkdir -p "$destPath"
if [ $? -ne 0 ]; then
Logger "Cannot create crypt storage path [$destPath]." "ERROR"
return 1
fi
fi
if [ ! -w "$destPath" ]; then if [ ! -w "$destPath" ]; then
Logger "Cannot write to crypt storage path [$destPath]." "ERROR" Logger "Cannot write to crypt storage path [$destPath]." "ERROR"
return 1 return 1
@ -1095,7 +1102,7 @@ function EncryptFiles {
fi fi
# /root/obackup-storage/files-pull//root/obackup-testdata/testData/dir rect ory/some file] to [/root/obackup-storage/crypt//dir rect ory/some file.obackup.gpg # /root/obackup-storage/files-pull//root/obackup-testdata/testData/dir rect ory/some file] to [/root/obackup-storage/crypt//dir rect ory/some file.obackup.gpg
Logger "Encrypting files in [$filePath]." "NOTICE" #WIP level = vERBOSe Logger "Encrypting files in [$filePath]." "VERBOSE"
while IFS= read -r -d $'\0' sourceFile; do while IFS= read -r -d $'\0' sourceFile; do
# Get path of sourcefile # Get path of sourcefile
path="$(dirname "$sourceFile")" path="$(dirname "$sourceFile")"
@ -1200,27 +1207,15 @@ function DecryptFiles {
} }
function Rsync { function Rsync {
local backupDirectory="${1}" # Which directory to backup local sourceDir="${1}" # Source directory
local destinationDir="${2}" # Destination directory
local recursive="${2:-true}" # Backup only files at toplevel of directory local recursive="${2:-true}" # Backup only files at toplevel of directory
__CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG
local fileStoragePath
local withoutCryptPath
local rsyncCmd local rsyncCmd
local retval local retval
if [ "$KEEP_ABSOLUTE_PATHS" != "no" ]; then
if [ "$ENCRYPTION" == "yes" ]; then
withoutCryptPath="${backupDirectory#$CRYPT_STORAGE}"
fileStoragePath=$(dirname "$FILE_STORAGE/${withoutCryptPath#/}")
else
fileStoragePath=$(dirname "$FILE_STORAGE/${backupDirectory#/}")
fi
else
fileStoragePath="$FILE_STORAGE"
fi
## Manage to backup recursive directories lists files only (not recursing into subdirectories) ## Manage to backup recursive directories lists files only (not recursing into subdirectories)
if [ $recursive == false ]; then if [ $recursive == false ]; then
# Fixes symlinks to directories in target cannot be deleted when backing up root directory without recursion, and excludes subdirectories # Fixes symlinks to directories in target cannot be deleted when backing up root directory without recursion, and excludes subdirectories
@ -1229,22 +1224,25 @@ function Rsync {
RSYNC_NO_RECURSE_ARGS="" RSYNC_NO_RECURSE_ARGS=""
fi fi
Logger "Beginning file backup of [$sourceDir] to [$destinationDir]." "VERBOSE"
# Creating subdirectories because rsync cannot handle multiple subdirectory creation # Creating subdirectories because rsync cannot handle multiple subdirectory creation
if [ "$BACKUP_TYPE" == "local" ]; then if [ "$BACKUP_TYPE" == "local" ]; then
_CreateDirectoryLocal "$fileStoragePath" _CreateDirectoryLocal "$destinationDir"
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$backupDirectory\" \"$fileStoragePath\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$sourceDir\" \"$destinationDir\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1"
elif [ "$BACKUP_TYPE" == "pull" ]; then elif [ "$BACKUP_TYPE" == "pull" ]; then
_CreateDirectoryLocal "$fileStoragePath" _CreateDirectoryLocal "$destinationDir"
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
backupDirectory=$(EscapeSpaces "$backupDirectory") sourceDir=$(EscapeSpaces "$sourceDir")
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$backupDirectory\" \"$fileStoragePath\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$sourceDir\" \"$destinationDir\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1"
elif [ "$BACKUP_TYPE" == "push" ]; then elif [ "$BACKUP_TYPE" == "push" ]; then
fileStoragePath=$(EscapeSpaces "$fileStoragePath") destinationDir=$(EscapeSpaces "$destinationDir")
_CreateDirectoryRemote "$fileStoragePath" _CreateDirectoryRemote "$destinationDir"
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$backupDirectory\" \"$REMOTE_USER@$REMOTE_HOST:$fileStoragePath\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$sourceDir\" \"$REMOTE_USER@$REMOTE_HOST:$destinationDir\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1"
fi fi
Logger "cmd: $rsyncCmd" "DEBUG" Logger "cmd: $rsyncCmd" "DEBUG"
@ -1252,7 +1250,7 @@ function Rsync {
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Failed to backup [$backupDirectory] to [$fileStoragePath]." "ERROR" Logger "Failed to backup [$sourceDir] to [$destinationDir]." "ERROR"
Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "ERROR" Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "ERROR"
else else
Logger "File backup succeed." "NOTICE" Logger "File backup succeed." "NOTICE"
@ -1266,46 +1264,66 @@ function FilesBackup {
local backupTask local backupTask
local backupTasks local backupTasks
local destinationDir
local withoutCryptPath
IFS=$PATH_SEPARATOR_CHAR read -r -a backupTasks <<< "$FILE_BACKUP_TASKS" IFS=$PATH_SEPARATOR_CHAR read -r -a backupTasks <<< "$FILE_BACKUP_TASKS"
for backupTask in "${backupTasks[@]}"; do for backupTask in "${backupTasks[@]}"; do
Logger "Beginning file backup of [$backupTask]." "NOTICE"
if [ "$KEEP_ABSOLUTE_PATHS" != "no" ]; then
destinationDir=$(dirname "$FILE_STORAGE/${backupTask#/}")
encryptDir="$FILE_STORAGE/${backupTask#/}"
else
destinationDir="$FILE_STORAGE"
encryptDir="$FILE_STORAGE"
fi
Logger "Beginning backup task [$backupTask]." "NOTICE"
if [ "$ENCRYPTION" == "yes" ] && ([ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]); then if [ "$ENCRYPTION" == "yes" ] && ([ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]); then
EncryptFiles "$backupTask" "$CRYPT_STORAGE" "$GPG_RECIPIENT" true true EncryptFiles "$backupTask" "$CRYPT_STORAGE" "$GPG_RECIPIENT" true true
if [ $? == 0 ]; then if [ $? == 0 ]; then
Rsync "$CRYPT_STORAGE/$backupTask" true Rsync "$CRYPT_STORAGE/$backupTask" "$destinationDir" true
else else
Logger "backup failed." "ERROR" Logger "backup failed." "ERROR"
fi fi
elif [ "$ENCRYPTION" == "yes" ] && [ "$BACKUP_TYPE" == "pull" ]; then elif [ "$ENCRYPTION" == "yes" ] && [ "$BACKUP_TYPE" == "pull" ]; then
Rsync "$backupTask" true Rsync "$backupTask" "$destinationDir" true
if [ $? == 0 ]; then if [ $? == 0 ]; then
EncryptFiles "$FILE_STORAGE/$backupTask" "$CRYPT_STORAGE" "$GPG_RECIPIENT" true false EncryptFiles "$encryptDir" "$CRYPT_STORAGE/$backupTask" "$GPG_RECIPIENT" true false
fi fi
else else
Rsync "$backupTask" true Rsync "$backupTask" "$destinationDir" true
fi fi
CheckTotalExecutionTime CheckTotalExecutionTime
done done
IFS=$PATH_SEPARATOR_CHAR read -r -a backupTasks <<< "$RECURSIVE_DIRECTORY_LIST" IFS=$PATH_SEPARATOR_CHAR read -r -a backupTasks <<< "$RECURSIVE_DIRECTORY_LIST"
for backupTask in "${backupTasks[@]}"; do for backupTask in "${backupTasks[@]}"; do
Logger "Beginning non recursive file backup of [$backupTask]." "NOTICE"
if [ "$KEEP_ABSOLUTE_PATHS" != "no" ]; then
destinationDir=$(dirname "$FILE_STORAGE/${backupTask#/}")
encryptDir="$FILE_STORAGE/${backupTask#/}"
else
destinationDir="$FILE_STORAGE"
encryptDir="$FILE_STORAGE"
fi
Logger "Beginning backup task [$backupTask]." "NOTICE"
if [ "$ENCRYPTION" == "yes" ] && ([ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]); then if [ "$ENCRYPTION" == "yes" ] && ([ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]); then
EncryptFiles "$backupTask" "$CRYPT_STORAGE" "$GPG_RECIPIENT" false true EncryptFiles "$backupTask" "$CRYPT_STORAGE" "$GPG_RECIPIENT" false true
if [ $? == 0 ]; then if [ $? == 0 ]; then
#TODO why was there another true param ?Rsync "$CRYPT_STORAGE/$backupTask" false true Rsync "$CRYPT_STORAGE/$backupTask" "$destinationDir" false
Rsync "$CRYPT_STORAGE/$backupTask" false
else else
Logger "backup failed." "ERROR" Logger "backup failed." "ERROR"
fi fi
elif [ "$ENCRYPTION" == "yes" ] && [ "$BACKUP_TYPE" == "pull" ]; then elif [ "$ENCRYPTION" == "yes" ] && [ "$BACKUP_TYPE" == "pull" ]; then
Rsync "$backupTask" false Rsync "$backupTask" "$destinationDir" false
if [ $? == 0 ]; then if [ $? == 0 ]; then
EncryptFiles "$FILE_STORAGE/$backupTask" "$CRYPT_STORAGE" "$GPG_RECIPIENT" false false EncryptFiles "$encryptDir" "$CRYPT_STORAGE/$backupTask" "$GPG_RECIPIENT" false false
fi fi
else else
Rsync "$backupTask" false Rsync "$backupTask" "$destinationDir" false
fi fi
CheckTotalExecutionTime CheckTotalExecutionTime
done done
@ -1313,23 +1331,31 @@ function FilesBackup {
IFS=$PATH_SEPARATOR_CHAR read -r -a backupTasks <<< "$FILE_RECURSIVE_BACKUP_TASKS" IFS=$PATH_SEPARATOR_CHAR read -r -a backupTasks <<< "$FILE_RECURSIVE_BACKUP_TASKS"
for backupTask in "${backupTasks[@]}"; do for backupTask in "${backupTasks[@]}"; do
# Backup sub directories of recursive directories # Backup sub directories of recursive directories
Logger "Beginning recursive file backup of [$backupTask]." "NOTICE"
if [ "$KEEP_ABSOLUTE_PATHS" != "no" ]; then
destinationDir=$(dirname "$FILE_STORAGE/${backupTask#/}")
encryptDir="$FILE_STORAGE/${backupTask#/}"
else
destinationDir="$FILE_STORAGE"
encryptDir="$FILE_STORAGE"
fi
Logger "Beginning backup task [$backupTask]." "NOTICE"
if [ "$ENCRYPTION" == "yes" ] && ([ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]); then if [ "$ENCRYPTION" == "yes" ] && ([ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]); then
EncryptFiles "$backupTask" "$CRYPT_STORAGE" "$GPG_RECIPIENT" true true EncryptFiles "$backupTask" "$CRYPT_STORAGE" "$GPG_RECIPIENT" true true
if [ $? == 0 ]; then if [ $? == 0 ]; then
#TODO same as above Rsync "$CRYPT_STORAGE/$backupTask" true true Rsync "$CRYPT_STORAGE/$backupTask" "$destinationDir" true
Rsync "$CRYPT_STORAGE/$backupTask" true #TODO Where is output checking here ?
# Where is output checking here ?
else else
Logger "backup failed." "ERROR" Logger "backup failed." "ERROR"
fi fi
elif [ "$ENCRYPTION" == "yes" ] && [ "$BACKUP_TYPE" == "pull" ]; then elif [ "$ENCRYPTION" == "yes" ] && [ "$BACKUP_TYPE" == "pull" ]; then
Rsync "$backupTask" true Rsync "$backupTask" "$destinationDir" true
if [ $? == 0 ]; then if [ $? == 0 ]; then
EncryptFiles "$FILE_STORAGE/$backupTask" "$CRYPT_STORAGE" "$GPG_RECIPIENT" true false EncryptFiles "$encryptDir" "$CRYPT_STORAGE/$backupTask" "$GPG_RECIPIENT" true false
fi fi
else else
Rsync "$backupTask" true Rsync "$backupTask" "$destinationDir" true
fi fi
CheckTotalExecutionTime CheckTotalExecutionTime
done done
@ -1506,7 +1532,7 @@ ENDSSH
} }
#TODO: test find cmd for backup rotation with regex on busybox / mac #TODO: test find cmd for backup rotation with regex on busybox / mac
function RotateBackups { function RotateBackups {
local backupPath="${1}" local backupPath="${1}"
local rotateCopies="${2}" local rotateCopies="${2}"
@ -1752,7 +1778,7 @@ if [ "$_ENCRYPT_MODE" == true ]; then
GetLocalOS GetLocalOS
InitLocalOSDependingSettings InitLocalOSDependingSettings
Logger "$DRY_WARNING$PROGRAM v$PROGRAM_VERSION encrypt mode begin." "ALWAYS" Logger "$DRY_WARNING$PROGRAM v$PROGRAM_VERSION encrypt mode begin." "ALWAYS"
EncryptFiles "$CRYPT_SOURCE" "$CRYPT_STORAGE" "$GPG_RECIPIENT" true false EncryptFiles "" "$CRYPT_SOURCE" "$CRYPT_STORAGE" "$GPG_RECIPIENT" true false
exit $? exit $?
fi fi