Linux/obackup
1
0
Fork 0
mirror of https://github.com/deajan/obackup.git synced 2025-05-10 20:23:41 +02:00
Code Issues Releases Wiki Activity

Added include patterns to backup

Browse Source
This commit is contained in:
deajan 2015-11-28 12:48:19 +01:00
parent a95c37e67f
commit 6cd1815ed9
5 changed files with 181 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -11,6 +11,7 @@ CHANGELOG
README: FreeBSD execution needs mailer (not found), sudo missing, bash needed, sed missing (see if StripQuotes mandatory) README: FreeBSD execution needs mailer (not found), sudo missing, bash needed, sed missing (see if StripQuotes mandatory)
! XX Dec 2015: obackup v2.0 released ! XX Dec 2015: obackup v2.0 released
- Revamped rsync patterns to allow include and exclude patterns
- Better SQL and file backup task separation (rotate copies and warnings are defined for sql and/or file) - Better SQL and file backup task separation (rotate copies and warnings are defined for sql and/or file)
- Added reverse backup, now backups can be local, pushed or pulled to or from a remote system - Added reverse backup, now backups can be local, pushed or pulled to or from a remote system
- Better fallback for SendAlert even if disk full - Better fallback for SendAlert even if disk full

75
dev/debug_obackup.sh
View File

@ -6,7 +6,7 @@ PROGRAM="obackup"
AUTHOR="(L) 2013-2015 by Orsiris de Jong" AUTHOR="(L) 2013-2015 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
PROGRAM_VERSION=2.0-pre PROGRAM_VERSION=2.0-pre
PROGRAM_BUILD=2015112002 PROGRAM_BUILD=2015112802
IS_STABLE=no IS_STABLE=no
FUNC_BUILD=2015111901 FUNC_BUILD=2015111901
@ -1662,17 +1662,17 @@ function Rsync {
# Creating subdirectories because rsync cannot handle multiple subdirectory creation # Creating subdirectories because rsync cannot handle multiple subdirectory creation
if [ "$BACKUP_TYPE" == "local" ]; then if [ "$BACKUP_TYPE" == "local" ]; then
_CreateDirectoryLocal "$file_storage_path" _CreateDirectoryLocal "$file_storage_path"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1"
elif [ "$BACKUP_TYPE" == "pull" ]; then elif [ "$BACKUP_TYPE" == "pull" ]; then
_CreateDirectoryLocal "$file_storage_path" _CreateDirectoryLocal "$file_storage_path"
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1"
elif [ "$BACKUP_TYPE" == "push" ]; then elif [ "$BACKUP_TYPE" == "push" ]; then
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
_CreateDirectoryRemote "$file_storage_path" _CreateDirectoryRemote "$file_storage_path"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$backup_directory\" \"$REMOTE_USER@$REMOTE_HOST:$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$backup_directory\" \"$REMOTE_USER@$REMOTE_HOST:$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1"
fi fi
Logger "cmd: $rsync_cmd" "DEBUG" Logger "cmd: $rsync_cmd" "DEBUG"
@ -1783,12 +1783,15 @@ function CheckTotalExecutionTime {
fi fi
} }
function RsyncExcludePattern { function RsyncPatternsAdd {
__CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG local pattern="${1}"
local pattern_type="${2}" # exclude or include
__CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
# Disable globbing so wildcards from exclusions do not get expanded # Disable globbing so wildcards from exclusions do not get expanded
set -f set -f
rest="$RSYNC_EXCLUDE_PATTERN" rest="$pattern"
while [ -n "$rest" ] while [ -n "$rest" ]
do do
# Take the string until first occurence until $PATH_SEPARATOR_CHAR # Take the string until first occurence until $PATH_SEPARATOR_CHAR
@ -1800,28 +1803,54 @@ function RsyncExcludePattern {
# Cut everything before the first occurence of $PATH_SEPARATOR_CHAR # Cut everything before the first occurence of $PATH_SEPARATOR_CHAR
rest=${rest#*$PATH_SEPARATOR_CHAR} rest=${rest#*$PATH_SEPARATOR_CHAR}
fi fi
if [ "$RSYNC_PATTERNS" == "" ]; then
if [ "$RSYNC_EXCLUDE" == "" ]; then RSYNC_PATTERNS="--"$pattern_type"=\"$str\""
RSYNC_EXCLUDE="--exclude=\"$str\""
else else
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$str\"" RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"=\"$str\""
fi fi
done done
set +f set +f
} }
function RsyncExcludeFrom { function RsyncPatternsFromAdd {
local pattern_from="${1}"
local pattern_type="${2}"
__CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
## Check if the exclude list has a full path, and if not, add the config file path if there is one
if [ "$(basename $pattern_from)" == "$pattern_from" ]; then
pattern_from="$(dirname $CONFIG_FILE)/$pattern_from"
fi
if [ -e "$pattern_from" ]; then
RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\""
fi
}
function RsyncPatterns {
__CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
if [ ! $RSYNC_EXCLUDE_FROM == "" ]; then if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then
## Check if the exclude list has a full path, and if not, add the config file path if there is one RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude"
if [ "$(basename $RSYNC_EXCLUDE_FROM)" == "$RSYNC_EXCLUDE_FROM" ]; then if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RSYNC_EXCLUDE_FROM=$(dirname $ConfigFile)/$RSYNC_EXCLUDE_FROM RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude"
fi fi
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include"
if [ -e $RSYNC_EXCLUDE_FROM ]; then if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\"" RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include"
fi fi
elif [ "$RSYNC_PATTERN_FIRST" == "include" ]; then
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include"
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include"
fi
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude"
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude"
fi
else
Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN"
fi fi
} }
@ -2069,7 +2098,7 @@ function Init {
if [ "$PARTIAL" == "yes" ]; then if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$PARTIAL_DIR\"" RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi fi
if [ "$DELETE_VANISHED_FILES" == "yes" ]; then if [ "$DELETE_VANISHED_FILES" == "yes" ]; then
@ -2122,10 +2151,8 @@ function Main {
if [ $_DRYRUN -ne 1 ] && [ "$ROTATE_FILE_BACKUPS" == "yes" ]; then if [ $_DRYRUN -ne 1 ] && [ "$ROTATE_FILE_BACKUPS" == "yes" ]; then
RotateBackups "$FILE_STORAGE" "$ROTATE_FILE_COPIES" RotateBackups "$FILE_STORAGE" "$ROTATE_FILE_COPIES"
fi fi
## Add Rsync exclude patterns ## Add Rsync include / exclude patterns
RsyncExcludePattern RsyncPatterns
## Add Rsync exclude from file
RsyncExcludeFrom
FilesBackup FilesBackup
fi fi
} }

111
dev/n_obackup.sh
View File

@ -6,7 +6,7 @@ PROGRAM="obackup"
AUTHOR="(L) 2013-2015 by Orsiris de Jong" AUTHOR="(L) 2013-2015 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
PROGRAM_VERSION=2.0-pre PROGRAM_VERSION=2.0-pre
PROGRAM_BUILD=2015112002 PROGRAM_BUILD=2015112802
IS_STABLE=no IS_STABLE=no
source "./ofunctions.sh" source "./ofunctions.sh"
@ -774,17 +774,17 @@ function Rsync {
# Creating subdirectories because rsync cannot handle multiple subdirectory creation # Creating subdirectories because rsync cannot handle multiple subdirectory creation
if [ "$BACKUP_TYPE" == "local" ]; then if [ "$BACKUP_TYPE" == "local" ]; then
_CreateDirectoryLocal "$file_storage_path" _CreateDirectoryLocal "$file_storage_path"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1"
elif [ "$BACKUP_TYPE" == "pull" ]; then elif [ "$BACKUP_TYPE" == "pull" ]; then
_CreateDirectoryLocal "$file_storage_path" _CreateDirectoryLocal "$file_storage_path"
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1"
elif [ "$BACKUP_TYPE" == "push" ]; then elif [ "$BACKUP_TYPE" == "push" ]; then
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
_CreateDirectoryRemote "$file_storage_path" _CreateDirectoryRemote "$file_storage_path"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$backup_directory\" \"$REMOTE_USER@$REMOTE_HOST:$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$backup_directory\" \"$REMOTE_USER@$REMOTE_HOST:$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1"
fi fi
Logger "cmd: $rsync_cmd" "DEBUG" Logger "cmd: $rsync_cmd" "DEBUG"
@ -895,46 +895,75 @@ function CheckTotalExecutionTime {
fi fi
} }
function RsyncExcludePattern { function RsyncPatternsAdd {
__CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG local pattern="${1}"
local pattern_type="${2}" # exclude or include
# Disable globbing so wildcards from exclusions do not get expanded __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
set -f
rest="$RSYNC_EXCLUDE_PATTERN"
while [ -n "$rest" ]
do
# Take the string until first occurence until $PATH_SEPARATOR_CHAR
str=${rest%%;*}
# Handle the last case
if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ]; then
rest=
else
# Cut everything before the first occurence of $PATH_SEPARATOR_CHAR
rest=${rest#*$PATH_SEPARATOR_CHAR}
fi
if [ "$RSYNC_EXCLUDE" == "" ]; then # Disable globbing so wildcards from exclusions do not get expanded
RSYNC_EXCLUDE="--exclude=\"$str\"" set -f
else rest="$pattern"
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$str\"" while [ -n "$rest" ]
fi do
done # Take the string until first occurence until $PATH_SEPARATOR_CHAR
set +f str=${rest%%;*}
# Handle the last case
if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ]; then
rest=
else
# Cut everything before the first occurence of $PATH_SEPARATOR_CHAR
rest=${rest#*$PATH_SEPARATOR_CHAR}
fi
if [ "$RSYNC_PATTERNS" == "" ]; then
RSYNC_PATTERNS="--"$pattern_type"=\"$str\""
else
RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"=\"$str\""
fi
done
set +f
} }
function RsyncExcludeFrom { function RsyncPatternsFromAdd {
__CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG local pattern_from="${1}"
local pattern_type="${2}"
if [ ! $RSYNC_EXCLUDE_FROM == "" ]; then __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
## Check if the exclude list has a full path, and if not, add the config file path if there is one
if [ "$(basename $RSYNC_EXCLUDE_FROM)" == "$RSYNC_EXCLUDE_FROM" ]; then
RSYNC_EXCLUDE_FROM=$(dirname $ConfigFile)/$RSYNC_EXCLUDE_FROM
fi
if [ -e $RSYNC_EXCLUDE_FROM ]; then ## Check if the exclude list has a full path, and if not, add the config file path if there is one
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\"" if [ "$(basename $pattern_from)" == "$pattern_from" ]; then
pattern_from="$(dirname $CONFIG_FILE)/$pattern_from"
fi
if [ -e "$pattern_from" ]; then
RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\""
fi
}
function RsyncPatterns {
__CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude"
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude"
fi fi
fi RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include"
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include"
fi
elif [ "$RSYNC_PATTERN_FIRST" == "include" ]; then
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include"
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include"
fi
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude"
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude"
fi
else
Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN"
fi
} }
function _RotateBackupsLocal { function _RotateBackupsLocal {
@ -1181,7 +1210,7 @@ function Init {
if [ "$PARTIAL" == "yes" ]; then if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$PARTIAL_DIR\"" RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi fi
if [ "$DELETE_VANISHED_FILES" == "yes" ]; then if [ "$DELETE_VANISHED_FILES" == "yes" ]; then
@ -1234,10 +1263,8 @@ function Main {
if [ $_DRYRUN -ne 1 ] && [ "$ROTATE_FILE_BACKUPS" == "yes" ]; then if [ $_DRYRUN -ne 1 ] && [ "$ROTATE_FILE_BACKUPS" == "yes" ]; then
RotateBackups "$FILE_STORAGE" "$ROTATE_FILE_COPIES" RotateBackups "$FILE_STORAGE" "$ROTATE_FILE_COPIES"
fi fi
## Add Rsync exclude patterns ## Add Rsync include / exclude patterns
RsyncExcludePattern RsyncPatterns
## Add Rsync exclude from file
RsyncExcludeFrom
FilesBackup FilesBackup
fi fi
} }

14
host_backup.conf
View File

@ -101,12 +101,20 @@ DIRECTORY_LIST="/var/named"
RECURSIVE_DIRECTORY_LIST="/home" RECURSIVE_DIRECTORY_LIST="/home"
RECURSIVE_EXCLUDE_LIST="/home/backupuser;/home/lost+found" RECURSIVE_EXCLUDE_LIST="/home/backupuser;/home/lost+found"
## Rsync exclude patterns, used by simple and division lists, separated by semicolons ## Rsync exclude / include order (the option set here will be set first, eg: include will make include then exclude patterns)
RSYNC_EXCLUDE_PATTERN="*/tmp;*/ftp/www/cache/cachefs;*/sessions" RSYNC_PATTERN_FIRST=include
## File that contains the list of directories or files to exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file. ## List of files / directories to incldue / exclude from sync on both sides (see rsync patterns, wildcards work).
## Paths are relative to sync dirs. List elements are separated by a semicolon.
RSYNC_INCLUDE_PATTERN=""
RSYNC_EXCLUDE_PATTERN=""
#RSYNC_EXCLUDE_PATTERN="tmp;archives"
## Files that contains lists of files / directories to include / exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
## This file has to be in the same directory as the config file ## This file has to be in the same directory as the config file
## Paths are relative to sync dirs. One element per line. ## Paths are relative to sync dirs. One element per line.
RSYNC_INCLUDE_FROM=""
RSYNC_EXCLUDE_FROM=""
#RSYNC_EXCLUDE_FROM="exclude.list" #RSYNC_EXCLUDE_FROM="exclude.list"
## List separator char. You may set an alternative separator char for your directories lists above. ## List separator char. You may set an alternative separator char for your directories lists above.

72
obackup.sh
View File

@ -6,7 +6,7 @@ PROGRAM="obackup"
AUTHOR="(L) 2013-2015 by Orsiris de Jong" AUTHOR="(L) 2013-2015 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
PROGRAM_VERSION=2.0-pre PROGRAM_VERSION=2.0-pre
PROGRAM_BUILD=2015112002 PROGRAM_BUILD=2015112802
IS_STABLE=no IS_STABLE=no
FUNC_BUILD=2015111901 FUNC_BUILD=2015111901
@ -1538,17 +1538,17 @@ function Rsync {
# Creating subdirectories because rsync cannot handle multiple subdirectory creation # Creating subdirectories because rsync cannot handle multiple subdirectory creation
if [ "$BACKUP_TYPE" == "local" ]; then if [ "$BACKUP_TYPE" == "local" ]; then
_CreateDirectoryLocal "$file_storage_path" _CreateDirectoryLocal "$file_storage_path"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1"
elif [ "$BACKUP_TYPE" == "pull" ]; then elif [ "$BACKUP_TYPE" == "pull" ]; then
_CreateDirectoryLocal "$file_storage_path" _CreateDirectoryLocal "$file_storage_path"
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1"
elif [ "$BACKUP_TYPE" == "push" ]; then elif [ "$BACKUP_TYPE" == "push" ]; then
CheckConnectivity3rdPartyHosts CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost CheckConnectivityRemoteHost
_CreateDirectoryRemote "$file_storage_path" _CreateDirectoryRemote "$file_storage_path"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$backup_directory\" \"$REMOTE_USER@$REMOTE_HOST:$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$backup_directory\" \"$REMOTE_USER@$REMOTE_HOST:$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1"
fi fi
Logger "cmd: $rsync_cmd" "DEBUG" Logger "cmd: $rsync_cmd" "DEBUG"
@ -1656,11 +1656,14 @@ function CheckTotalExecutionTime {
fi fi
} }
function RsyncExcludePattern { function RsyncPatternsAdd {
local pattern="${1}"
local pattern_type="${2}" # exclude or include
# Disable globbing so wildcards from exclusions do not get expanded # Disable globbing so wildcards from exclusions do not get expanded
set -f set -f
rest="$RSYNC_EXCLUDE_PATTERN" rest="$pattern"
while [ -n "$rest" ] while [ -n "$rest" ]
do do
# Take the string until first occurence until $PATH_SEPARATOR_CHAR # Take the string until first occurence until $PATH_SEPARATOR_CHAR
@ -1672,27 +1675,52 @@ function RsyncExcludePattern {
# Cut everything before the first occurence of $PATH_SEPARATOR_CHAR # Cut everything before the first occurence of $PATH_SEPARATOR_CHAR
rest=${rest#*$PATH_SEPARATOR_CHAR} rest=${rest#*$PATH_SEPARATOR_CHAR}
fi fi
if [ "$RSYNC_PATTERNS" == "" ]; then
if [ "$RSYNC_EXCLUDE" == "" ]; then RSYNC_PATTERNS="--"$pattern_type"=\"$str\""
RSYNC_EXCLUDE="--exclude=\"$str\""
else else
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$str\"" RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"=\"$str\""
fi fi
done done
set +f set +f
} }
function RsyncExcludeFrom { function RsyncPatternsFromAdd {
local pattern_from="${1}"
local pattern_type="${2}"
if [ ! $RSYNC_EXCLUDE_FROM == "" ]; then
## Check if the exclude list has a full path, and if not, add the config file path if there is one
if [ "$(basename $RSYNC_EXCLUDE_FROM)" == "$RSYNC_EXCLUDE_FROM" ]; then
RSYNC_EXCLUDE_FROM=$(dirname $ConfigFile)/$RSYNC_EXCLUDE_FROM
fi
if [ -e $RSYNC_EXCLUDE_FROM ]; then ## Check if the exclude list has a full path, and if not, add the config file path if there is one
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\"" if [ "$(basename $pattern_from)" == "$pattern_from" ]; then
pattern_from="$(dirname $CONFIG_FILE)/$pattern_from"
fi
if [ -e "$pattern_from" ]; then
RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\""
fi
}
function RsyncPatterns {
if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude"
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude"
fi fi
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include"
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include"
fi
elif [ "$RSYNC_PATTERN_FIRST" == "include" ]; then
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include"
if [ "$RSYNC_INCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include"
fi
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude"
if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude"
fi
else
Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN"
fi fi
} }
@ -1931,7 +1959,7 @@ function Init {
if [ "$PARTIAL" == "yes" ]; then if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$PARTIAL_DIR\"" RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi fi
if [ "$DELETE_VANISHED_FILES" == "yes" ]; then if [ "$DELETE_VANISHED_FILES" == "yes" ]; then
@ -1983,10 +2011,8 @@ function Main {
if [ $_DRYRUN -ne 1 ] && [ "$ROTATE_FILE_BACKUPS" == "yes" ]; then if [ $_DRYRUN -ne 1 ] && [ "$ROTATE_FILE_BACKUPS" == "yes" ]; then
RotateBackups "$FILE_STORAGE" "$ROTATE_FILE_COPIES" RotateBackups "$FILE_STORAGE" "$ROTATE_FILE_COPIES"
fi fi
## Add Rsync exclude patterns ## Add Rsync include / exclude patterns
RsyncExcludePattern RsyncPatterns
## Add Rsync exclude from file
RsyncExcludeFrom
FilesBackup FilesBackup
fi fi
} }