diff --git a/dev/debug_obackup.sh b/dev/debug_obackup.sh index b95a28c..78656a7 100755 --- a/dev/debug_obackup.sh +++ b/dev/debug_obackup.sh @@ -6,8 +6,8 @@ PROGRAM="obackup" AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" -PROGRAM_VERSION=2.1-beta5 -PROGRAM_BUILD=2018091201 +PROGRAM_VERSION=2.1-RC1 +PROGRAM_BUILD=201809301 IS_STABLE=no #### Execution order #__WITH_PARANOIA_DEBUG @@ -34,8 +34,8 @@ IS_STABLE=no # RsyncPatterns #__WITH_PARANOIA_DEBUG # FilesBackup #__WITH_PARANOIA_DEBUG -_OFUNCTIONS_VERSION=2.3.0-dev -_OFUNCTIONS_BUILD=2018070902 +_OFUNCTIONS_VERSION=2.3.0-RC1 +_OFUNCTIONS_BUILD=2018093001 _OFUNCTIONS_BOOTSTRAP=true ## To use in a program, define the following variables: @@ -47,8 +47,6 @@ _OFUNCTIONS_BOOTSTRAP=true ## _LOGGER_ERR_ONLY=true/false ## _LOGGER_PREFIX="date"/"time"/"" -#TODO: global WAIT_FOR_TASK_COMPLETION_id instead of callerName has to be backported to ParallelExec and osync / obackup / pmocr ocde - ## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel ## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP @@ -300,7 +298,7 @@ function Logger { if [ "$_LOGGER_PREFIX" == "time" ]; then prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then - prefix="$(date) - " + prefix="$(date '+%Y-%m-%d %H:%M:%S') - " else prefix="" fi @@ -2126,8 +2124,8 @@ function InitRemoteOSDependingSettings { REMOTE_STAT_CTIME_MTIME_CMD="stat -c \\\"%n;%Z;%Y\\\"" fi - ## Set rsync default arguments - RSYNC_ARGS="-rltD -8" + ## Set rsync default arguments (complete with -r or -d depending on recursivity later) + RSYNC_DEFAULT_ARGS="ltD -8" if [ "$_DRYRUN" == true ]; then RSYNC_DRY_ARG="-n" DRY_WARNING="/!\ DRY RUN " @@ -2165,43 +2163,46 @@ function InitRemoteOSDependingSettings { fi if [ "$RSYNC_COMPRESS" == "yes" ]; then if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -zz --skip-compress=gz/xz/lz/lzma/lzo/rz/jpg/mp3/mp4/7z/bz2/rar/zip/sfark/s7z/ace/apk/arc/cab/dmg/jar/kgb/lzh/lha/lzx/pak/sfx" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -zz --skip-compress=gz/xz/lz/lzma/lzo/rz/jpg/mp3/mp4/7z/bz2/rar/zip/sfark/s7z/ace/apk/arc/cab/dmg/jar/kgb/lzh/lha/lzx/pak/sfx" else Logger "Disabling compression skips on synchronization on [$LOCAL_OS] due to lack of support." "NOTICE" fi fi if [ "$COPY_SYMLINKS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -L" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -L" fi if [ "$KEEP_DIRLINKS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -K" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -K" fi if [ "$RSYNC_OPTIONAL_ARGS" != "" ]; then - RSYNC_ARGS=$RSYNC_ARGS" "$RSYNC_OPTIONAL_ARGS + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" "$RSYNC_OPTIONAL_ARGS fi if [ "$PRESERVE_HARDLINKS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -H" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -H" fi if [ "$CHECKSUM" == "yes" ]; then RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum" fi if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then - RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --bwlimit=$BANDWIDTH" fi if [ "$PARTIAL" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\"" fi if [ "$DELTA_COPIES" != "no" ]; then - RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --no-whole-file" else - RSYNC_ARGS=$RSYNC_ARGS" --whole-file" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --whole-file" fi # Set compression options again after we know what remote OS we are dealing with SetCompression + + RSYNC_DEFAULT_ARGS="-r $RSYNC_DEFAULT_ARGS" + RSYNC_DEFAULT_NONRECURSIVE_ARGS="-d $RSYNC_DEFAULT_ARGS" } ## IFS debug function @@ -4158,30 +4159,32 @@ function Rsync { local rsyncCmd local retval + local rsyncArgs + ## Manage to backup recursive directories lists files only (not recursing into subdirectories) if [ $recursive == false ]; then - # Fixes symlinks to directories in target cannot be deleted when backing up root directory without recursion, and excludes subdirectories - RSYNC_NO_RECURSE_ARGS=" -k --exclude=*/*/" + # Fixes symlinks to directories in target cannot be deleted when backing up root directory without recursion + rsyncArgs="$RSYNC_DEFAULT_NONRECURSIVE_ARGS -k" else - RSYNC_NO_RECURSE_ARGS="" + rsyncArgs="$RSYNC_DEFAULT_ARGS" fi # Creating subdirectories because rsync cannot handle multiple subdirectory creation if [ "$BACKUP_TYPE" == "local" ]; then _CreateDirectoryLocal "$destinationDir" - rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$sourceDir\" \"$destinationDir\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" + rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $rsyncArgs $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$sourceDir\" \"$destinationDir\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" elif [ "$BACKUP_TYPE" == "pull" ]; then _CreateDirectoryLocal "$destinationDir" CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost sourceDir=$(EscapeSpaces "$sourceDir") - rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$sourceDir\" \"$destinationDir\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" + rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $rsyncArgs $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$sourceDir\" \"$destinationDir\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" elif [ "$BACKUP_TYPE" == "push" ]; then destinationDir=$(EscapeSpaces "$destinationDir") _CreateDirectoryRemote "$destinationDir" CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$sourceDir\" \"$REMOTE_USER@$REMOTE_HOST:$destinationDir\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" + rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $rsyncARgs $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$sourceDir\" \"$REMOTE_USER@$REMOTE_HOST:$destinationDir\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" fi Logger "Launching command [$rsyncCmd]." "DEBUG" diff --git a/install.sh b/install.sh index c54b6e1..9d28282 100755 --- a/install.sh +++ b/install.sh @@ -206,7 +206,7 @@ function Logger { if [ "$_LOGGER_PREFIX" == "time" ]; then prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then - prefix="$(date) - " + prefix="$(date '+%Y-%m-%d %H:%M:%S') - " else prefix="" fi diff --git a/obackup.sh b/obackup.sh index 1dbb856..6294aa0 100755 --- a/obackup.sh +++ b/obackup.sh @@ -6,13 +6,13 @@ PROGRAM="obackup" AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" -PROGRAM_VERSION=2.1-beta5 -PROGRAM_BUILD=2018091201 +PROGRAM_VERSION=2.1-RC1 +PROGRAM_BUILD=201809301 IS_STABLE=no -_OFUNCTIONS_VERSION=2.3.0-dev -_OFUNCTIONS_BUILD=2018070902 +_OFUNCTIONS_VERSION=2.3.0-RC1 +_OFUNCTIONS_BUILD=2018093001 _OFUNCTIONS_BOOTSTRAP=true ## To use in a program, define the following variables: @@ -24,8 +24,6 @@ _OFUNCTIONS_BOOTSTRAP=true ## _LOGGER_ERR_ONLY=true/false ## _LOGGER_PREFIX="date"/"time"/"" -#TODO: global WAIT_FOR_TASK_COMPLETION_id instead of callerName has to be backported to ParallelExec and osync / obackup / pmocr ocde - ## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel ## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP @@ -267,7 +265,7 @@ function Logger { if [ "$_LOGGER_PREFIX" == "time" ]; then prefix="TIME: $SECONDS - " elif [ "$_LOGGER_PREFIX" == "date" ]; then - prefix="$(date) - " + prefix="$(date '+%Y-%m-%d %H:%M:%S') - " else prefix="" fi @@ -1981,8 +1979,8 @@ function InitRemoteOSDependingSettings { REMOTE_STAT_CTIME_MTIME_CMD="stat -c \\\"%n;%Z;%Y\\\"" fi - ## Set rsync default arguments - RSYNC_ARGS="-rltD -8" + ## Set rsync default arguments (complete with -r or -d depending on recursivity later) + RSYNC_DEFAULT_ARGS="ltD -8" if [ "$_DRYRUN" == true ]; then RSYNC_DRY_ARG="-n" DRY_WARNING="/!\ DRY RUN " @@ -2020,43 +2018,46 @@ function InitRemoteOSDependingSettings { fi if [ "$RSYNC_COMPRESS" == "yes" ]; then if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -zz --skip-compress=gz/xz/lz/lzma/lzo/rz/jpg/mp3/mp4/7z/bz2/rar/zip/sfark/s7z/ace/apk/arc/cab/dmg/jar/kgb/lzh/lha/lzx/pak/sfx" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -zz --skip-compress=gz/xz/lz/lzma/lzo/rz/jpg/mp3/mp4/7z/bz2/rar/zip/sfark/s7z/ace/apk/arc/cab/dmg/jar/kgb/lzh/lha/lzx/pak/sfx" else Logger "Disabling compression skips on synchronization on [$LOCAL_OS] due to lack of support." "NOTICE" fi fi if [ "$COPY_SYMLINKS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -L" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -L" fi if [ "$KEEP_DIRLINKS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -K" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -K" fi if [ "$RSYNC_OPTIONAL_ARGS" != "" ]; then - RSYNC_ARGS=$RSYNC_ARGS" "$RSYNC_OPTIONAL_ARGS + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" "$RSYNC_OPTIONAL_ARGS fi if [ "$PRESERVE_HARDLINKS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -H" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -H" fi if [ "$CHECKSUM" == "yes" ]; then RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum" fi if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then - RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --bwlimit=$BANDWIDTH" fi if [ "$PARTIAL" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\"" fi if [ "$DELTA_COPIES" != "no" ]; then - RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --no-whole-file" else - RSYNC_ARGS=$RSYNC_ARGS" --whole-file" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --whole-file" fi # Set compression options again after we know what remote OS we are dealing with SetCompression + + RSYNC_DEFAULT_ARGS="-r $RSYNC_DEFAULT_ARGS" + RSYNC_DEFAULT_NONRECURSIVE_ARGS="-d $RSYNC_DEFAULT_ARGS" } ## IFS debug function @@ -3951,30 +3952,32 @@ function Rsync { local rsyncCmd local retval + local rsyncArgs + ## Manage to backup recursive directories lists files only (not recursing into subdirectories) if [ $recursive == false ]; then - # Fixes symlinks to directories in target cannot be deleted when backing up root directory without recursion, and excludes subdirectories - RSYNC_NO_RECURSE_ARGS=" -k --exclude=*/*/" + # Fixes symlinks to directories in target cannot be deleted when backing up root directory without recursion + rsyncArgs="$RSYNC_DEFAULT_NONRECURSIVE_ARGS -k" else - RSYNC_NO_RECURSE_ARGS="" + rsyncArgs="$RSYNC_DEFAULT_ARGS" fi # Creating subdirectories because rsync cannot handle multiple subdirectory creation if [ "$BACKUP_TYPE" == "local" ]; then _CreateDirectoryLocal "$destinationDir" - rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$sourceDir\" \"$destinationDir\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" + rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $rsyncArgs $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"$RSYNC_PATH\" \"$sourceDir\" \"$destinationDir\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" elif [ "$BACKUP_TYPE" == "pull" ]; then _CreateDirectoryLocal "$destinationDir" CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost sourceDir=$(EscapeSpaces "$sourceDir") - rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$sourceDir\" \"$destinationDir\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" + rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $rsyncArgs $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$REMOTE_USER@$REMOTE_HOST:$sourceDir\" \"$destinationDir\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" elif [ "$BACKUP_TYPE" == "push" ]; then destinationDir=$(EscapeSpaces "$destinationDir") _CreateDirectoryRemote "$destinationDir" CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $RSYNC_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$sourceDir\" \"$REMOTE_USER@$REMOTE_HOST:$destinationDir\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" + rsyncCmd="$(type -p $RSYNC_EXECUTABLE) $rsyncARgs $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $RSYNC_NO_RECURSE_ARGS $RSYNC_DELETE $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --rsync-path=\"env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"$sourceDir\" \"$REMOTE_USER@$REMOTE_HOST:$destinationDir\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" fi Logger "Launching command [$rsyncCmd]." "DEBUG"