diff --git a/dev/common_batch.sh b/dev/common_batch.sh index 508c8ac..1569b4b 100755 --- a/dev/common_batch.sh +++ b/dev/common_batch.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash SUBPROGRAM=[prgname] PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones -AUTHOR="(L) 2013-2018 by Orsiris de Jong" +AUTHOR="(L) 2013-2020 by Orsiris de Jong" CONTACT="http://www.netpower.fr - ozy@netpower.fr" -PROGRAM_BUILD=2018100201 +PROGRAM_BUILD=2020031502 ## Runs an osync /obackup instance for every conf file found ## If an instance fails, run it again if time permits @@ -34,12 +34,11 @@ elif [ -w /var/tmp ]; then else RUN_DIR=. fi - -trap TrapQuit TERM EXIT HUP QUIT - # No need to edit under this line ############################################################## include #### Logger SUBSET #### +include #### CleanUp SUBSET #### +include #### GenericTrapQuit SUBSET #### function CheckEnvironment { ## osync / obackup executable full path can be set here if it cannot be found on the system @@ -129,6 +128,8 @@ function Usage { exit 128 } +trap GenericTrapQuit TERM EXIT HUP QUIT + opts="" for i in "$@" do diff --git a/dev/common_install.sh b/dev/common_install.sh index 02c89c3..6bec72c 100755 --- a/dev/common_install.sh +++ b/dev/common_install.sh @@ -10,7 +10,7 @@ PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" SSH_FILTER="ssh_filter.sh" -SCRIPT_BUILD=2019052001 +SCRIPT_BUILD=2020031502 INSTANCE_ID="installer-$SCRIPT_BUILD" ## osync / obackup / pmocr / zsnap install script @@ -27,62 +27,6 @@ _STATS=1 ACTION="install" FAKEROOT="" -function GetCommandlineArguments { - for i in "$@"; do - case $i in - --prefix=*) - FAKEROOT="${i##*=}" - ;; - --silent) - _LOGGER_SILENT=true - ;; - --no-stats) - _STATS=0 - ;; - --remove) - ACTION="uninstall" - ;; - --help|-h|-?) - Usage - ;; - *) - Logger "Unknown option '$i'" "ERROR" - Usage - exit - ;; - esac - done -} - -GetCommandlineArguments "$@" - -CONF_DIR=$FAKEROOT/etc/$PROGRAM -BIN_DIR="$FAKEROOT/usr/local/bin" -SERVICE_DIR_INIT=$FAKEROOT/etc/init.d -# Should be /usr/lib/systemd/system, but /lib/systemd/system exists on debian & rhel / fedora -SERVICE_DIR_SYSTEMD_SYSTEM=$FAKEROOT/lib/systemd/system -SERVICE_DIR_SYSTEMD_USER=$FAKEROOT/etc/systemd/user -SERVICE_DIR_OPENRC=$FAKEROOT/etc/init.d - -if [ "$PROGRAM" == "osync" ]; then - SERVICE_NAME="osync-srv" - TARGET_HELPER_SERVICE_NAME="osync-target-helper-srv" - - TARGET_HELPER_SERVICE_FILE_INIT="$TARGET_HELPER_SERVICE_NAME" - TARGET_HELPER_SERVICE_FILE_SYSTEMD_SYSTEM="$TARGET_HELPER_SERVICE_NAME@.service" - TARGET_HELPER_SERVICE_FILE_SYSTEMD_USER="$TARGET_HELPER_SERVICE_NAME@.service.user" - TARGET_HELPER_SERVICE_FILE_OPENRC="$TARGET_HELPER_SERVICE_NAME-openrc" -elif [ "$PROGRAM" == "pmocr" ]; then - SERVICE_NAME="pmocr-srv" -fi - -SERVICE_FILE_INIT="$SERVICE_NAME" -SERVICE_FILE_SYSTEMD_SYSTEM="$SERVICE_NAME@.service" -SERVICE_FILE_SYSTEMD_USER="$SERVICE_NAME@.service.user" -SERVICE_FILE_OPENRC="$SERVICE_NAME-openrc" - -## Generic code - ## Default log file if [ -w "$FAKEROOT/var/log" ]; then LOG_FILE="$FAKEROOT/var/log/$PROGRAM-install.log" @@ -95,6 +39,8 @@ fi include #### UrlEncode SUBSET #### include #### GetLocalOS SUBSET #### include #### GetConfFileValue SUBSET #### +include #### CleanUp SUBSET #### +include #### GenericTrapQuit SUBSET #### function SetLocalOSSettings { USER=root @@ -195,15 +141,14 @@ function CopyFile { local overwrite="${8:-false}" local userGroup="" - local oldFileName if [ "$destFileName" == "" ]; then destFileName="$sourceFileName" fi if [ -f "$destPath/$destFileName" ] && [ $overwrite == false ]; then - destfileName="$sourceFileName.new" - Logger "Copying [$sourceFileName] to [$destPath/$destFilename]." "NOTICE" + destFileName="$sourceFileName.new" + Logger "Copying [$sourceFileName] to [$destPath/$destFileName]." "NOTICE" fi cp "$sourcePath/$sourceFileName" "$destPath/$destFileName" @@ -291,11 +236,11 @@ function CopyServiceFiles { fi if [ -f "$SCRIPT_PATH/$TARGET_HELPER_SERVICE_FILE_SYSTEMD_SYSTEM" ]; then - CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$TARGET_HELPER_SERVICE_FILE_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" "" "" "" true + CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$TARGET_HELPER_SERVICE_FILE_SYSTEMD_SYSTEM" "$TARGET_HELPER_SERVICE_FILE_SYSTEMD_SYSTEM" "" "" "" true Logger "Created optional service [$TARGET_HELPER_SERVICE_NAME] with same specifications as below." "NOTICE" fi if [ -f "$SCRIPT_PATH/$TARGET_HELPER_SERVICE_FILE_SYSTEMD_USER" ]; then - CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$TARGET_HELPER_SERVICE_FILE_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true + CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$TARGET_HELPER_SERVICE_FILE_SYSTEMD_USER" "$TARGET_HELPER_SERVICE_FILE_SYSTEMD_USER" "" "" "" true fi @@ -307,7 +252,7 @@ function CopyServiceFiles { #CreateDir "$SERVICE_DIR_INIT" CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "$SERVICE_FILE_INIT" "755" "" "" true if [ -f "$SCRIPT_PATH/$TARGET_HELPER_SERVICE_FILE_INIT" ]; then - CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$TARGET_HELPER_SERVICE_FILE_INIT" "$SERVICE_FILE_INIT" "755" "" "" true + CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$TARGET_HELPER_SERVICE_FILE_INIT" "$TARGET_HELPER_SERVICE_FILE_INIT" "755" "" "" true Logger "Created optional service [$TARGET_HELPER_SERVICE_NAME] with same specifications as below." "NOTICE" fi Logger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]." "NOTICE" @@ -317,7 +262,7 @@ function CopyServiceFiles { # Rename service to usual service file CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_OPENRC" "$SERVICE_FILE_OPENRC" "$SERVICE_FILE_INIT" "755" "" "" true if [ -f "$SCRPT_PATH/$TARGET_HELPER_SERVICE_FILE_OPENRC" ]; then - CopyFile "$SCRIPT_PATH" "$TARGET_HELPER_SERVICE_DIR_OPENRC" "$SERVICE_FILE_OPENRC" "$SERVICE_FILE_INIT" "755" "" "" true + CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_OPENRC" "$TARGET_HELPER_SERVICE_FILE_OPENRC" "$TARGET_HELPER_SERVICE_FILE_OPENRC" "755" "" "" true Logger "Created optional service [$TARGET_HELPER_SERVICE_NAME] with same specifications as below." "NOTICE" fi Logger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_OPENRC]." "NOTICE" @@ -377,6 +322,10 @@ function RemoveAll { RemoveFile "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_USER" RemoveFile "$SERVICE_DIR_INIT/$SERVICE_FILE_INIT" + RemoveFile "$TARGET_HELPER_SERVICE_DIR_SYSTEMD_SYSTEM/$SERVICE_FILE_SYSTEMD_SYSTEM" + RemoveFile "$TARGET_HELPER_SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_USER" + RemoveFile "$TARGET_HELPER_SERVICE_DIR_INIT/$SERVICE_FILE_INIT" + Logger "Skipping configuration files in [$CONF_DIR]. You may remove this directory manually." "NOTICE" } @@ -390,26 +339,65 @@ function Usage { exit 127 } -function TrapQuit { - local exitcode=0 - - # Get ERROR / WARN alert flags from subprocesses that call Logger - if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then - WARN_ALERT=true - exitcode=2 - fi - if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then - ERROR_ALERT=true - exitcode=1 - fi - - CleanUp - exit $exitcode -} - ############################## Script entry point -trap TrapQuit TERM EXIT HUP QUIT +function GetCommandlineArguments { + for i in "$@"; do + case $i in + --prefix=*) + FAKEROOT="${i##*=}" + ;; + --silent) + _LOGGER_SILENT=true + ;; + --no-stats) + _STATS=0 + ;; + --remove) + ACTION="uninstall" + ;; + --help|-h|-?) + Usage + ;; + *) + Logger "Unknown option '$i'" "ERROR" + Usage + exit + ;; + esac + done +} + +GetCommandlineArguments "$@" + +CONF_DIR=$FAKEROOT/etc/$PROGRAM +BIN_DIR="$FAKEROOT/usr/local/bin" +SERVICE_DIR_INIT=$FAKEROOT/etc/init.d +# Should be /usr/lib/systemd/system, but /lib/systemd/system exists on debian & rhel / fedora +SERVICE_DIR_SYSTEMD_SYSTEM=$FAKEROOT/lib/systemd/system +SERVICE_DIR_SYSTEMD_USER=$FAKEROOT/etc/systemd/user +SERVICE_DIR_OPENRC=$FAKEROOT/etc/init.d + +if [ "$PROGRAM" == "osync" ]; then + SERVICE_NAME="osync-srv" + TARGET_HELPER_SERVICE_NAME="osync-target-helper-srv" + + TARGET_HELPER_SERVICE_FILE_INIT="$TARGET_HELPER_SERVICE_NAME" + TARGET_HELPER_SERVICE_FILE_SYSTEMD_SYSTEM="$TARGET_HELPER_SERVICE_NAME@.service" + TARGET_HELPER_SERVICE_FILE_SYSTEMD_USER="$TARGET_HELPER_SERVICE_NAME@.service.user" + TARGET_HELPER_SERVICE_FILE_OPENRC="$TARGET_HELPER_SERVICE_NAME-openrc" +elif [ "$PROGRAM" == "pmocr" ]; then + SERVICE_NAME="pmocr-srv" +fi + +SERVICE_FILE_INIT="$SERVICE_NAME" +SERVICE_FILE_SYSTEMD_SYSTEM="$SERVICE_NAME@.service" +SERVICE_FILE_SYSTEMD_USER="$SERVICE_NAME@.service.user" +SERVICE_FILE_OPENRC="$SERVICE_NAME-openrc" + +## Generic code + +trap GenericTrapQuit TERM EXIT HUP QUIT if [ ! -w "$(dirname $LOG_FILE)" ]; then echo "Cannot write to log [$(dirname $LOG_FILE)]." diff --git a/dev/merge.sh b/dev/merge.sh index 472ea97..4b0140b 100755 --- a/dev/merge.sh +++ b/dev/merge.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -## MERGE 2019022601 +## MERGE 2020031501 ## Merges ofunctions.sh and n_program.sh into program.sh ## Adds installer @@ -61,6 +61,8 @@ function __PREPROCESSOR_Constants { '#### SetConfFileValue SUBSET ####' '#### CheckRFC822 SUBSET ####' '#### CleanUp SUBSET ####' + '#### GenericTrapQuit SUBSET ####' + '#### FileMove SUBSET ####' ) } diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 506de3a..d7a5c96 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -## Generic and highly portable bash functions written in 2013-2019 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr +## Generic and highly portable bash functions written in 2013-2020 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr #TODO: ExecTasks postponed arrays / files grow a lot. Consider having them "rolling" (cleaned at numberOfEvents) #TODO: command line arguments don't take -AaqV for example @@ -30,8 +30,8 @@ #### OFUNCTIONS FULL SUBSET #### #### OFUNCTIONS MINI SUBSET #### #### OFUNCTIONS MICRO SUBSET #### -_OFUNCTIONS_VERSION=2.3.0-dev-postRC2 -_OFUNCTIONS_BUILD=2019080904 +_OFUNCTIONS_VERSION=2.3.0-RC4 +_OFUNCTIONS_BUILD=2020031502 #### _OFUNCTIONS_BOOTSTRAP SUBSET #### _OFUNCTIONS_BOOTSTRAP=true #### _OFUNCTIONS_BOOTSTRAP SUBSET END #### @@ -421,6 +421,26 @@ function KillAllChilds { return $errorcount } +#### GenericTrapQuit SUBSET #### +function GenericTrapQuit { + local exitcode=0 + + # Get ERROR / WARN alert flags from subprocesses that call Logger + if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then + WARN_ALERT=true + exitcode=2 + fi + if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then + ERROR_ALERT=true + exitcode=1 + fi + + CleanUp + exit $exitcode +} + +#### TrapQuit SUBSET END #### + #### CleanUp SUBSET #### function CleanUp { # Exit controlmaster before it's socket gets deleted @@ -439,26 +459,9 @@ function CleanUp { rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" fi } + #### CleanUp SUBSET END #### -function GenericTrapQuit { - local exitcode=0 - - # Get ERROR / WARN alert flags from subprocesses that call Logger - if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then - WARN_ALERT=true - exitcode=2 - fi - if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then - ERROR_ALERT=true - exitcode=1 - fi - - CleanUp - exit $exitcode -} - - #### OFUNCTIONS MICRO SUBSET END #### # osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending @@ -561,7 +564,7 @@ function SendEmail { fi done else - Logger "No valid email adresses given." "WARN" + Logger "No valid email addresses given." "WARN" return 1 fi @@ -776,7 +779,7 @@ function _PerfProfiler { #__WITH_PARANOIA_DEBUG _OFUNCTIONS_SPINNER="|/-\\" function Spinner { - if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ]; then + if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ] || [ "$_SYNC_ON_CHANGES" == "initiator" ] || [ "$_SYNC_ON_CHANGES" == "target" ] ; then return 0 else printf " [%c] \b\b\b\b\b\b" "$_OFUNCTIONS_SPINNER" @@ -905,7 +908,7 @@ function ExecTasks { local currentCommand # Variable containing currently processed command local currentCommandCondition # Variable containing currently processed conditional command local commandsArrayPid=() # Array containing commands indexed by pids - local commandsArrayOutput=() # Array contining command results indexed by pids + local commandsArrayOutput=() # Array containing command results indexed by pids local postponedRetryCount=0 # Number of current postponed commands retries local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once) local postponedCounter=0 @@ -2021,6 +2024,11 @@ function PreInit { SSH_OPTS="$SSH_OPTS -o ControlMaster=auto -o ControlPersist=yes -o ControlPath=\"$RUN_DIR/$PROGRAM.ctrlm.%r@%h.$SCRIPT_PID.$TSTAMP\"" fi + ## Optional SSH arguments + if [ "$SSH_OPTIONAL_ARGS" != "" ]; then + SSH_OPTS="$SSH_OPTS $SSH_OPTIONAL_ARGS" + fi + ## Support for older config files without RSYNC_EXECUTABLE option if [ "$RSYNC_EXECUTABLE" == "" ]; then RSYNC_EXECUTABLE=rsync @@ -2234,7 +2242,7 @@ function InitRemoteOSDependingSettings { fi if [ "$RSYNC_COMPRESS" == true ]; then if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then - RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -zz --skip-compress=gz/xz/lz/lzma/lzo/rz/jpg/mp3/mp4/7z/bz2/rar/zip/sfark/s7z/ace/apk/arc/cab/dmg/jar/kgb/lzh/lha/lzx/pak/sfx" + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -zz --skip-compress=3fr/3g2/3gp/3gpp/7z/aac/ace/amr/apk/appx/appxbundle/arc/arj/arw/asf/avi/bz/bz2/cab/cr2/crypt[5678]/dat/dcr/deb/dmg/drc/ear/erf/flac/flv/gif/gpg/gz/iiq/jar/jp2/jpeg/jpg/h26[45]/k25/kdc/kgb/lha/lz/lzma/lzo/lzx/m4[apv]/mef/mkv/mos/mov/mp[34]/mpeg/mp[gv]/msi/nef/oga/ogg/ogv/opus/orf/pak/pef/png/qt/rar/r[0-9][0-9]/rz/rpm/rw2/rzip/s7z/sfark/sfx/sr2/srf/svgz/t[gb]z/tlz/txz/vob/wim/wma/wmv/xz/zip" else Logger "Disabling compression skips on synchronization on [$LOCAL_OS] due to lack of support." "NOTICE" fi @@ -2418,4 +2426,25 @@ function WildcardFileExists () { fi } +# Some MacOS versions might loose file ownsership when using mv from /tmp dir (see #175) +# This is a "mv" function wrapper that helps out with macOS +#### FileMove SUBSET #### +function FileMove () { + local source="${1}" + local dest="${2}" + + # If file is symlink or OS is not Mac, just make a standard mv + if [ -L "$source" ] || [ "$LOCAL_OS" != "MacOSX" ]; then + mv -f "$source" "$dest" + return $? + elif [ -w "$source" ]; then + [ -f "$dest" ] && rm -f "$dest" + cp -p "$source" "$dest" && rm -f "$source" + return $? + else + return -1 + fi +} +#### FileMove SUBSET END #### + #### OFUNCTIONS FULL SUBSET END ####