diff --git a/CHANGELOG.md b/CHANGELOG.md index bd810be..f08ab88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ CHANGELOG README: FreeBSD execution needs mailer (not found), sudo missing, bash needed, sed missing (see if StripQuotes mandatory) ! XX Dec 2015: obackup v2.0 released +- Revamped rsync patterns to allow include and exclude patterns - Better SQL and file backup task separation (rotate copies and warnings are defined for sql and/or file) - Added reverse backup, now backups can be local, pushed or pulled to or from a remote system - Better fallback for SendAlert even if disk full diff --git a/dev/debug_obackup.sh b/dev/debug_obackup.sh index 87df7c4..f54816d 100755 --- a/dev/debug_obackup.sh +++ b/dev/debug_obackup.sh @@ -6,7 +6,7 @@ PROGRAM="obackup" AUTHOR="(L) 2013-2015 by Orsiris de Jong" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" PROGRAM_VERSION=2.0-pre -PROGRAM_BUILD=2015112002 +PROGRAM_BUILD=2015112802 IS_STABLE=no FUNC_BUILD=2015111901 @@ -1662,17 +1662,17 @@ function Rsync { # Creating subdirectories because rsync cannot handle multiple subdirectory creation if [ "$BACKUP_TYPE" == "local" ]; then _CreateDirectoryLocal "$file_storage_path" - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" elif [ "$BACKUP_TYPE" == "pull" ]; then _CreateDirectoryLocal "$file_storage_path" CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" elif [ "$BACKUP_TYPE" == "push" ]; then CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost _CreateDirectoryRemote "$file_storage_path" - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$backup_directory\" \"$REMOTE_USER@$REMOTE_HOST:$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$backup_directory\" \"$REMOTE_USER@$REMOTE_HOST:$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" fi Logger "cmd: $rsync_cmd" "DEBUG" @@ -1783,12 +1783,15 @@ function CheckTotalExecutionTime { fi } -function RsyncExcludePattern { - __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG +function RsyncPatternsAdd { + local pattern="${1}" + local pattern_type="${2}" # exclude or include + + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG # Disable globbing so wildcards from exclusions do not get expanded set -f - rest="$RSYNC_EXCLUDE_PATTERN" + rest="$pattern" while [ -n "$rest" ] do # Take the string until first occurence until $PATH_SEPARATOR_CHAR @@ -1800,28 +1803,54 @@ function RsyncExcludePattern { # Cut everything before the first occurence of $PATH_SEPARATOR_CHAR rest=${rest#*$PATH_SEPARATOR_CHAR} fi - - if [ "$RSYNC_EXCLUDE" == "" ]; then - RSYNC_EXCLUDE="--exclude=\"$str\"" + if [ "$RSYNC_PATTERNS" == "" ]; then + RSYNC_PATTERNS="--"$pattern_type"=\"$str\"" else - RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$str\"" + RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"=\"$str\"" fi done set +f } -function RsyncExcludeFrom { +function RsyncPatternsFromAdd { + local pattern_from="${1}" + local pattern_type="${2}" + + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + ## Check if the exclude list has a full path, and if not, add the config file path if there is one + if [ "$(basename $pattern_from)" == "$pattern_from" ]; then + pattern_from="$(dirname $CONFIG_FILE)/$pattern_from" + fi + + if [ -e "$pattern_from" ]; then + RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\"" + fi +} + +function RsyncPatterns { __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG - if [ ! $RSYNC_EXCLUDE_FROM == "" ]; then - ## Check if the exclude list has a full path, and if not, add the config file path if there is one - if [ "$(basename $RSYNC_EXCLUDE_FROM)" == "$RSYNC_EXCLUDE_FROM" ]; then - RSYNC_EXCLUDE_FROM=$(dirname $ConfigFile)/$RSYNC_EXCLUDE_FROM + if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then + RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude" + if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude" fi - - if [ -e $RSYNC_EXCLUDE_FROM ]; then - RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\"" + RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include" + if [ "$RSYNC_INCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include" fi + elif [ "$RSYNC_PATTERN_FIRST" == "include" ]; then + RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include" + if [ "$RSYNC_INCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include" + fi + RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude" + if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude" + fi + else + Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN" fi } @@ -2069,7 +2098,7 @@ function Init { if [ "$PARTIAL" == "yes" ]; then RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" - RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$PARTIAL_DIR\"" + RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\"" fi if [ "$DELETE_VANISHED_FILES" == "yes" ]; then @@ -2122,10 +2151,8 @@ function Main { if [ $_DRYRUN -ne 1 ] && [ "$ROTATE_FILE_BACKUPS" == "yes" ]; then RotateBackups "$FILE_STORAGE" "$ROTATE_FILE_COPIES" fi - ## Add Rsync exclude patterns - RsyncExcludePattern - ## Add Rsync exclude from file - RsyncExcludeFrom + ## Add Rsync include / exclude patterns + RsyncPatterns FilesBackup fi } diff --git a/dev/n_obackup.sh b/dev/n_obackup.sh index 3463f23..02b2d4a 100755 --- a/dev/n_obackup.sh +++ b/dev/n_obackup.sh @@ -6,7 +6,7 @@ PROGRAM="obackup" AUTHOR="(L) 2013-2015 by Orsiris de Jong" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" PROGRAM_VERSION=2.0-pre -PROGRAM_BUILD=2015112002 +PROGRAM_BUILD=2015112802 IS_STABLE=no source "./ofunctions.sh" @@ -774,17 +774,17 @@ function Rsync { # Creating subdirectories because rsync cannot handle multiple subdirectory creation if [ "$BACKUP_TYPE" == "local" ]; then _CreateDirectoryLocal "$file_storage_path" - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" elif [ "$BACKUP_TYPE" == "pull" ]; then _CreateDirectoryLocal "$file_storage_path" CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" elif [ "$BACKUP_TYPE" == "push" ]; then CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost _CreateDirectoryRemote "$file_storage_path" - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$backup_directory\" \"$REMOTE_USER@$REMOTE_HOST:$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$backup_directory\" \"$REMOTE_USER@$REMOTE_HOST:$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" fi Logger "cmd: $rsync_cmd" "DEBUG" @@ -895,46 +895,75 @@ function CheckTotalExecutionTime { fi } -function RsyncExcludePattern { - __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG +function RsyncPatternsAdd { + local pattern="${1}" + local pattern_type="${2}" # exclude or include - # Disable globbing so wildcards from exclusions do not get expanded - set -f - rest="$RSYNC_EXCLUDE_PATTERN" - while [ -n "$rest" ] - do - # Take the string until first occurence until $PATH_SEPARATOR_CHAR - str=${rest%%;*} - # Handle the last case - if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ]; then - rest= - else - # Cut everything before the first occurence of $PATH_SEPARATOR_CHAR - rest=${rest#*$PATH_SEPARATOR_CHAR} - fi + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG - if [ "$RSYNC_EXCLUDE" == "" ]; then - RSYNC_EXCLUDE="--exclude=\"$str\"" - else - RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$str\"" - fi - done - set +f + # Disable globbing so wildcards from exclusions do not get expanded + set -f + rest="$pattern" + while [ -n "$rest" ] + do + # Take the string until first occurence until $PATH_SEPARATOR_CHAR + str=${rest%%;*} + # Handle the last case + if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ]; then + rest= + else + # Cut everything before the first occurence of $PATH_SEPARATOR_CHAR + rest=${rest#*$PATH_SEPARATOR_CHAR} + fi + if [ "$RSYNC_PATTERNS" == "" ]; then + RSYNC_PATTERNS="--"$pattern_type"=\"$str\"" + else + RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"=\"$str\"" + fi + done + set +f } -function RsyncExcludeFrom { - __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG +function RsyncPatternsFromAdd { + local pattern_from="${1}" + local pattern_type="${2}" - if [ ! $RSYNC_EXCLUDE_FROM == "" ]; then - ## Check if the exclude list has a full path, and if not, add the config file path if there is one - if [ "$(basename $RSYNC_EXCLUDE_FROM)" == "$RSYNC_EXCLUDE_FROM" ]; then - RSYNC_EXCLUDE_FROM=$(dirname $ConfigFile)/$RSYNC_EXCLUDE_FROM - fi + __CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG - if [ -e $RSYNC_EXCLUDE_FROM ]; then - RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\"" + ## Check if the exclude list has a full path, and if not, add the config file path if there is one + if [ "$(basename $pattern_from)" == "$pattern_from" ]; then + pattern_from="$(dirname $CONFIG_FILE)/$pattern_from" + fi + + if [ -e "$pattern_from" ]; then + RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\"" + fi +} + +function RsyncPatterns { + __CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG + + if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then + RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude" + if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude" fi - fi + RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include" + if [ "$RSYNC_INCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include" + fi + elif [ "$RSYNC_PATTERN_FIRST" == "include" ]; then + RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include" + if [ "$RSYNC_INCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include" + fi + RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude" + if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude" + fi + else + Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN" + fi } function _RotateBackupsLocal { @@ -1181,7 +1210,7 @@ function Init { if [ "$PARTIAL" == "yes" ]; then RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" - RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$PARTIAL_DIR\"" + RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\"" fi if [ "$DELETE_VANISHED_FILES" == "yes" ]; then @@ -1234,10 +1263,8 @@ function Main { if [ $_DRYRUN -ne 1 ] && [ "$ROTATE_FILE_BACKUPS" == "yes" ]; then RotateBackups "$FILE_STORAGE" "$ROTATE_FILE_COPIES" fi - ## Add Rsync exclude patterns - RsyncExcludePattern - ## Add Rsync exclude from file - RsyncExcludeFrom + ## Add Rsync include / exclude patterns + RsyncPatterns FilesBackup fi } diff --git a/host_backup.conf b/host_backup.conf index 4ea583f..a91eac9 100644 --- a/host_backup.conf +++ b/host_backup.conf @@ -101,12 +101,20 @@ DIRECTORY_LIST="/var/named" RECURSIVE_DIRECTORY_LIST="/home" RECURSIVE_EXCLUDE_LIST="/home/backupuser;/home/lost+found" -## Rsync exclude patterns, used by simple and division lists, separated by semicolons -RSYNC_EXCLUDE_PATTERN="*/tmp;*/ftp/www/cache/cachefs;*/sessions" +## Rsync exclude / include order (the option set here will be set first, eg: include will make include then exclude patterns) +RSYNC_PATTERN_FIRST=include -## File that contains the list of directories or files to exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file. +## List of files / directories to incldue / exclude from sync on both sides (see rsync patterns, wildcards work). +## Paths are relative to sync dirs. List elements are separated by a semicolon. +RSYNC_INCLUDE_PATTERN="" +RSYNC_EXCLUDE_PATTERN="" +#RSYNC_EXCLUDE_PATTERN="tmp;archives" + +## Files that contains lists of files / directories to include / exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file. ## This file has to be in the same directory as the config file ## Paths are relative to sync dirs. One element per line. +RSYNC_INCLUDE_FROM="" +RSYNC_EXCLUDE_FROM="" #RSYNC_EXCLUDE_FROM="exclude.list" ## List separator char. You may set an alternative separator char for your directories lists above. diff --git a/obackup.sh b/obackup.sh index 6063792..38559df 100755 --- a/obackup.sh +++ b/obackup.sh @@ -6,7 +6,7 @@ PROGRAM="obackup" AUTHOR="(L) 2013-2015 by Orsiris de Jong" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" PROGRAM_VERSION=2.0-pre -PROGRAM_BUILD=2015112002 +PROGRAM_BUILD=2015112802 IS_STABLE=no FUNC_BUILD=2015111901 @@ -1538,17 +1538,17 @@ function Rsync { # Creating subdirectories because rsync cannot handle multiple subdirectory creation if [ "$BACKUP_TYPE" == "local" ]; then _CreateDirectoryLocal "$file_storage_path" - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" elif [ "$BACKUP_TYPE" == "pull" ]; then _CreateDirectoryLocal "$file_storage_path" CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$backup_directory\" \"$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" elif [ "$BACKUP_TYPE" == "push" ]; then CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost _CreateDirectoryRemote "$file_storage_path" - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$backup_directory\" \"$REMOTE_USER@$REMOTE_HOST:$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_NO_RECURSE_ARGS --stats $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$backup_directory\" \"$REMOTE_USER@$REMOTE_HOST:$file_storage_path\" > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1" fi Logger "cmd: $rsync_cmd" "DEBUG" @@ -1656,11 +1656,14 @@ function CheckTotalExecutionTime { fi } -function RsyncExcludePattern { +function RsyncPatternsAdd { + local pattern="${1}" + local pattern_type="${2}" # exclude or include + # Disable globbing so wildcards from exclusions do not get expanded set -f - rest="$RSYNC_EXCLUDE_PATTERN" + rest="$pattern" while [ -n "$rest" ] do # Take the string until first occurence until $PATH_SEPARATOR_CHAR @@ -1672,27 +1675,52 @@ function RsyncExcludePattern { # Cut everything before the first occurence of $PATH_SEPARATOR_CHAR rest=${rest#*$PATH_SEPARATOR_CHAR} fi - - if [ "$RSYNC_EXCLUDE" == "" ]; then - RSYNC_EXCLUDE="--exclude=\"$str\"" + if [ "$RSYNC_PATTERNS" == "" ]; then + RSYNC_PATTERNS="--"$pattern_type"=\"$str\"" else - RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$str\"" + RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"=\"$str\"" fi done set +f } -function RsyncExcludeFrom { +function RsyncPatternsFromAdd { + local pattern_from="${1}" + local pattern_type="${2}" - if [ ! $RSYNC_EXCLUDE_FROM == "" ]; then - ## Check if the exclude list has a full path, and if not, add the config file path if there is one - if [ "$(basename $RSYNC_EXCLUDE_FROM)" == "$RSYNC_EXCLUDE_FROM" ]; then - RSYNC_EXCLUDE_FROM=$(dirname $ConfigFile)/$RSYNC_EXCLUDE_FROM - fi - if [ -e $RSYNC_EXCLUDE_FROM ]; then - RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\"" + ## Check if the exclude list has a full path, and if not, add the config file path if there is one + if [ "$(basename $pattern_from)" == "$pattern_from" ]; then + pattern_from="$(dirname $CONFIG_FILE)/$pattern_from" + fi + + if [ -e "$pattern_from" ]; then + RSYNC_PATTERNS="$RSYNC_PATTERNS --"$pattern_type"-from=\"$pattern_from\"" + fi +} + +function RsyncPatterns { + + if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then + RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude" + if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude" fi + RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include" + if [ "$RSYNC_INCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include" + fi + elif [ "$RSYNC_PATTERN_FIRST" == "include" ]; then + RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN" "include" + if [ "$RSYNC_INCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM" "include" + fi + RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN" "exclude" + if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM" "exclude" + fi + else + Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN" fi } @@ -1931,7 +1959,7 @@ function Init { if [ "$PARTIAL" == "yes" ]; then RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" - RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$PARTIAL_DIR\"" + RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\"" fi if [ "$DELETE_VANISHED_FILES" == "yes" ]; then @@ -1983,10 +2011,8 @@ function Main { if [ $_DRYRUN -ne 1 ] && [ "$ROTATE_FILE_BACKUPS" == "yes" ]; then RotateBackups "$FILE_STORAGE" "$ROTATE_FILE_COPIES" fi - ## Add Rsync exclude patterns - RsyncExcludePattern - ## Add Rsync exclude from file - RsyncExcludeFrom + ## Add Rsync include / exclude patterns + RsyncPatterns FilesBackup fi }