From 73c0191db6b4d112a964938665e8ad77a1b7be4e Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 24 Nov 2014 10:28:23 +0100 Subject: [PATCH] Added rsync exclude list support from osync --- CHANGELOG.md | 11 +++++++++-- host_backup.conf | 13 +++++++++---- obackup.sh | 48 +++++++++++++++++++++++++++++++----------------- 3 files changed, 49 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6306eef..890f6bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,17 +20,24 @@ KNOWN ISSUES - Dry mode creates dirs - Dry mode does not try mysqldump +UNDER WORK +---------- + +- Commands like cp should have their stderr redirected to log file +- Mysqldump must be checked for not telling success if a table is damaged + + CHANGELOG --------- +- Added Rsync exclude files suppport from osync - Fixed another issue with existing symlinks to directories on target on non recursive backups - Fixed remaining rsync -E option preventing obackup to work correctly on MacOS X - Fixed an issue with existing symlinks to directories on target - Prevent changed IFS to make ping commands fail - Added RotateCopies execution time (spinner support) - redirect stderr for mysqldump to catch problems - !-and other commands (cp ?) + verify is not telling success if table damaged -!- Moved msys specific code to Init(Local|Remote)OSSettings (halfway done, still need the WaitForTaskCompletion code to moove) +- Moved msys specific code to Init(Local|Remote)OSSettings except in TrapQuit that needs to work at any moment - Added support for multithreaded gzip (if pigz is installed) - Merged back changes from osync codebase - Enhanced debugging diff --git a/host_backup.conf b/host_backup.conf index 785276d..d2055a6 100755 --- a/host_backup.conf +++ b/host_backup.conf @@ -2,7 +2,7 @@ ###### Remote (or local) backup script for files & databases ###### (L) 2013 by Ozy de Jong (www.netpower.fr) -###### Config file rev 0903201401 +###### Config file rev 2411201401 ## ---------- GENERAL BACKUP OPTIONS @@ -82,9 +82,8 @@ DATABASES_LIST="" SOFT_MAX_EXEC_TIME_DB_TASK=3600 HARD_MAX_EXEC_TIME_DB_TASK=7200 -## Preferred SQL dump compression. Can be set to xz, lzma or gzip. -## Generally, xz level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable. -COMPRESSION_PROGRAM=xz +## Preferred SQL dump compression. Compression methods can be xz, lzma, pigz or gzip (will fallback from xz to gzip depending if available) +## Generally, level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable. COMPRESSION_LEVEL=3 ## SQL Dump compression should be done on remote side but can also be done locally to lower remote system usage (will take more bandwidth, check for ssh compression) @@ -108,6 +107,12 @@ DIRECTORIES_RECURSE_EXCLUDE_LIST="/home/backupuser;/home/lost+found" ## Rsync exclude patterns, used by simple and division lists RSYNC_EXCLUDE_PATTERN="*/tmp;*/ftp/www/cache/cachefs;*/sessions" +## File that contains the list of directories or files to exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file. +## This file has to be in the same directory as the config file +## Paths are relative to sync dirs. One element per line. +RSYNC_EXCLUDE_FROM="" +#RSYNC_EXCLUDE_FROM="exclude.list" + ## List separator char. You may set an alternative seperator char for your directories lists above. PATH_SEPARATOR_CHAR=";" diff --git a/obackup.sh b/obackup.sh index 43d7b7b..3f25feb 100755 --- a/obackup.sh +++ b/obackup.sh @@ -5,7 +5,7 @@ AUTHOR="(L) 2013-2014 by Orsiris \"Ozy\" de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.84preRC4 -PROGRAM_BUILD=1911201402 +PROGRAM_BUILD=2411201401 ## type doesn't work on platforms other than linux (bash). If if doesn't work, always assume output is not a zero exitcode if ! type -p "$BASH" > /dev/null @@ -417,12 +417,6 @@ function WaitForTaskCompletion { soft_alert=0 SECONDS_BEGIN=$SECONDS - if [ "$LOCAL_OS" == "msys" ] - then - PROCESS_TEST_CMD="ps -a | awk '{\$1=\$1}\$1' | awk '{print \$1}' | grep $1" - else - PROCESS_TEST_CMD="ps -p$1" - fi while eval $PROCESS_TEST_CMD > /dev/null do Spinner @@ -973,6 +967,23 @@ function RsyncExcludePattern IFS=$OLD_IFS } +function RsyncExcludeFrom +{ + if [ ! $RSYNC_EXCLUDE_FROM == "" ] + then + ## Check if the exclude list has a full path, and if not, add the config file path if there is one + if [ "$(basename $RSYNC_EXCLUDE_FROM)" == "$RSYNC_EXCLUDE_FROM" ] + then + RSYNC_EXCLUDE_FROM=$(dirname $ConfigFile)/$RSYNC_EXCLUDE_FROM + fi + + if [ -e $RSYNC_EXCLUDE_FROM ] + then + RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\"" + fi + fi +} + function Rsync { i="$(StripQuotes $1)" @@ -1283,12 +1294,19 @@ function Init function InitLocalOSSettings { - ## If running Msys, find command of windows is used instead of msys one + ## If running under Msys, some commands don't run the same way + ## Using mingw version of find instead of windows one + ## Getting running processes is quite different + ## Ping command isn't the same if [ "$LOCAL_OS" == "msys" ] then FIND_CMD=$(dirname $BASH)/find + PROCESS_TEST_CMD="ps -a | awk '{\$1=\$1}\$1' | awk '{print \$1}' | grep $1" + PING_CMD="ping -n 2" else FIND_CMD=find + PROCESS_TEST_CMD="ps -p$1" + PING_CMD="ping -c 2 -i .2" fi ## Stat command has different syntax on Linux and FreeBSD/MacOSX @@ -1298,14 +1316,6 @@ function InitLocalOSSettings else STAT_CMD="stat --format %y" fi - - ## Ping command has different syntax on Msys and others - if [ "$LOCAL_OS" == "msys" ] - then - PING_CMD="ping -n 2" - else - PING_CMD="ping -c 2 -i .2" - fi } function InitRemoteOSSettings @@ -1366,7 +1376,11 @@ function Main RotateBackups $LOCAL_FILE_STORAGE fi fi - RsyncExcludePattern + ## Add Rsync exclude patterns + RsyncExcludePattern + ## Add Rsync exclude from file + RsyncExcludeFrom + FilesBackup fi # Be a happy sysadmin (and drink a coffee ? Nahh... it's past midnight.)