1 #!/usr/bin/env bash 2 PROGRAM="Osync" # Rsync based two way sync engine with fault tolerance 3 AUTHOR="(L) 2013-2015 by Orsiris \"Ozy\" de Jong" 4 CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" 5 PROGRAM_VERSION=1.1-dev 6 PROGRAM_BUILD=2015073101 7 ## type doesn't work on platforms other than linux (bash). If if doesn't work, always assume output is not a zero exitcode 8 if ! type -p "$BASH" > /dev/null 9 then 10 echo "Please run this script only with bash shell. Tested on bash >= 3.2" 11 exit 127 12 fi 13 ## allow debugging from command line with preceding ocsync with DEBUG=yes 14 if [ ! "$DEBUG" == "yes" ] 15 then 16 DEBUG=no 17 SLEEP_TIME=.1 18 else 19 SLEEP_TIME=3 20 fi 21 SCRIPT_PID=$$ 22 LOCAL_USER=$(whoami) 23 LOCAL_HOST=$(hostname) 24 ## Default log file until config file is loaded 25 if [ -w /var/log ] 26 then 27 LOG_FILE=/var/log/osync.log 28 else 29 LOG_FILE=./osync.log 30 fi 31 ## Default directory where to store temporary run files 32 if [ -w /tmp ] 33 then 34 RUN_DIR=/tmp 35 elif [ -w /var/tmp ] 36 then 37 RUN_DIR=/var/tmp 38 else 39 RUN_DIR=. 40 fi 41 ## Working directory. Will keep current file states, backups and soft deleted files. 42 OSYNC_DIR=".osync_workdir" 43 ## Log a state message every $KEEP_LOGGING seconds. Should not be equal to soft or hard execution time so your log won't be unnecessary big. 44 KEEP_LOGGING=1801 45 ## Correct output of sort command (language agnostic sorting) 46 export LC_ALL=C 47 ALERT_LOG_FILE=$RUN_DIR/osync_lastlog 48 function Dummy 49 { 50 sleep .1 51 } 52 function Log 53 { 54 if [ $sync_on_changes -eq 1 ] 55 then 56 prefix="$(date) - " 57 else 58 prefix="TIME: $SECONDS - " 59 fi 60 echo -e "$prefix$1" >> "$LOG_FILE" 61 if [ $silent -eq 0 ] 62 then 63 echo -e "$prefix$1" 64 fi 65 } 66 function LogError 67 { 68 Log "$1" 69 error_alert=1 70 } 71 function LogDebug 72 { 73 if [ "$DEBUG" == "yes" ] 74 then 75 Log "$1" 76 fi 77 } 78 function TrapError 79 { 80 local JOB="$0" 81 local LINE="$1" 82 local CODE="${2:-1}" 83 if [ $silent -eq 0 ] 84 then 85 echo -e " /!\ ERROR in ${JOB}: Near line ${LINE}, exit code ${CODE}" 86 fi 87 } 88 function TrapStop 89 { 90 if [ $soft_stop -eq 0 ] 91 then 92 LogError " /!\ WARNING: Manual exit of osync is really not recommended. Sync will be in inconsistent state." 93 LogError " /!\ WARNING: If you are sure, please hit CTRL+C another time to quit." 94 soft_stop=1 95 return 1 96 fi 97 if [ $soft_stop -eq 1 ] 98 then 99 LogError " /!\ WARNING: CTRL+C hit twice. Quitting osync. Please wait..." 100 soft_stop=2 101 exit 1 102 fi 103 } 104 function TrapQuit 105 { 106 if [ $error_alert -ne 0 ] 107 then 108 if [ "$DEBUG" != "yes" ] 109 then 110 SendAlert 111 else 112 Log "Debug mode, no alert mail will be sent." 113 fi 114 UnlockDirectories 115 CleanUp 116 LogError "Osync finished with errors." 117 exitcode=1 118 else 119 UnlockDirectories 120 CleanUp 121 Log "Osync finished." 122 exitcode=0 123 fi 124 if ps -p $child_pid > /dev/null 2>&1 125 then 126 kill -9 $child_pid 127 fi 128 if ps -p $sub_pid > /dev/null 2>&1 129 then 130 kill -9 $sub_pid 131 fi 132 exit $exitcode 133 } 134 function Spinner 135 { 136 if [ $silent -eq 1 ] 137 then 138 return 1 139 fi 140 case $toggle 141 in 142 1) 143 echo -n $1" \ " 144 echo -ne "\r" 145 toggle="2" 146 ;; 147 2) 148 echo -n $1" | " 149 echo -ne "\r" 150 toggle="3" 151 ;; 152 3) 153 echo -n $1" / " 154 echo -ne "\r" 155 toggle="4" 156 ;; 157 *) 158 echo -n $1" - " 159 echo -ne "\r" 160 toggle="1" 161 ;; 162 esac 163 } 164 function EscapeSpaces 165 { 166 echo $(echo "$1" | sed 's/ /\\ /g') 167 } 168 function CleanUp 169 { 170 if [ "$DEBUG" != "yes" ] 171 then 172 rm -f $RUN_DIR/osync_*_$SCRIPT_PID 173 fi 174 } 175 function SendAlert 176 { 177 if [ "$quick_sync" == "2" ] 178 then 179 Log "Current task is a quicksync task. Will not send any alert." 180 return 0 181 fi 182 eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE" 183 MAIL_ALERT_MSG=$MAIL_ALERT_MSG$'\n\n'$(tail -n 25 "$LOG_FILE") 184 if type -p mutt > /dev/null 2>&1 185 then 186 echo $MAIL_ALERT_MSG | $(type -p mutt) -x -s "Sync alert for $SYNC_ID" $DESTINATION_MAILS -a "$ALERT_LOG_FILE" 187 if [ $? != 0 ] 188 then 189 Log "WARNING: Cannot send alert email via $(type -p mutt) !!!" 190 else 191 Log "Sent alert mail using mutt." 192 fi 193 elif type -p mail > /dev/null 2>&1 194 then 195 echo $MAIL_ALERT_MSG | $(type -p mail) -a "$ALERT_LOG_FILE" -s "Sync alert for $SYNC_ID" $DESTINATION_MAILS 196 if [ $? != 0 ] 197 then 198 Log "WARNING: Cannot send alert email via $(type -p mail) with attachments !!!" 199 echo $MAIL_ALERT_MSG | $(type -p mail) -s "Sync alert for $SYNC_ID" $DESTINATION_MAILS 200 if [ $? != 0 ] 201 then 202 Log "WARNING: Cannot send alert email via $(type -p mail) without attachments !!!" 203 else 204 Log "Sent alert mail using mail command without attachment." 205 fi 206 else 207 Log "Sent alert mail using mail command." 208 fi 209 elif type -p sendemail > /dev/null 2>&1 210 then 211 if [ "$SMTP_USER" != "" ] && [ "$SMTP_PASSWORD" != "" ] 212 then 213 $SMTP_OPTIONS="-xu $SMTP_USER -xp $SMTP_PASSWORD" 214 else 215 $SMTP_OPTIONS="" 216 fi 217 $(type -p sendemail) -f $SENDER_MAIL -t $DESTINATION_MAILS -u "Backup alert for $BACKUP_ID" -m "$MAIL_ALERT_MSG" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1 218 if [ $? != 0 ] 219 then 220 Log "WARNING: Cannot send alert email via $(type -p sendemail) !!!" 221 else 222 Log "Sent alert mail using sendemail command without attachment." 223 fi 224 else 225 Log "WARNING: Cannot send alert email (no mutt / mail present) !!!" 226 return 1 227 fi 228 if [ -f "$ALERT_LOG_FILE" ] 229 then 230 rm "$ALERT_LOG_FILE" 231 fi 232 } 233 function LoadConfigFile 234 { 235 if [ ! -f "$1" ] 236 then 237 LogError "Cannot load configuration file [$1]. Sync cannot start." 238 exit 1 239 elif [[ "$1" != *".conf" ]] 240 then 241 LogError "Wrong configuration file supplied [$1]. Sync cannot start." 242 exit 1 243 else 244 egrep '^#|^[^ ]*=[^;&]*' "$1" > "$RUN_DIR/osync_config_$SCRIPT_PID" 245 source "$RUN_DIR/osync_config_$SCRIPT_PID" 246 fi 247 } 248 function CheckEnvironment 249 { 250 if [ "$REMOTE_SYNC" == "yes" ] 251 then 252 if ! type -p ssh > /dev/null 2>&1 253 then 254 LogError "ssh not present. Cannot start sync." 255 return 1 256 fi 257 fi 258 if ! type -p rsync > /dev/null 2>&1 259 then 260 LogError "rsync not present. Sync cannot start." 261 return 1 262 fi 263 } 264 function GetLocalOS 265 { 266 LOCAL_OS_VAR=$(uname -spio 2>&1) 267 if [ $? != 0 ] 268 then 269 LOCAL_OS_VAR=$(uname -v 2>&1) 270 if [ $? != 0 ] 271 then 272 LOCAL_OS_VAR=($uname) 273 fi 274 fi 275 case $LOCAL_OS_VAR in 276 *"Linux"*) 277 LOCAL_OS="Linux" 278 ;; 279 *"BSD"*) 280 LOCAL_OS="BSD" 281 ;; 282 *"MINGW32"*) 283 LOCAL_OS="msys" 284 ;; 285 *"Darwin"*) 286 LOCAL_OS="MacOSX" 287 ;; 288 *) 289 LogError "Running on >> $LOCAL_OS_VAR << not supported. Please report to the author." 290 exit 1 291 ;; 292 esac 293 LogDebug "Local OS: [$LOCAL_OS_VAR]." 294 } 295 function GetRemoteOS 296 { 297 if [ "$REMOTE_SYNC" == "yes" ] 298 then 299 CheckConnectivity3rdPartyHosts 300 CheckConnectivityRemoteHost 301 eval "$SSH_CMD \"uname -spio\" > $RUN_DIR/osync_remote_os_$SCRIPT_PID 2>&1" & 302 child_pid=$! 303 WaitForTaskCompletion $child_pid 120 240 304 retval=$? 305 if [ $retval != 0 ] 306 then 307 eval "$SSH_CMD \"uname -v\" > $RUN_DIR/osync_remote_os_$SCRIPT_PID 2>&1" & 308 child_pid=$! 309 WaitForTaskCompletion $child_pid 120 240 310 retval=$? 311 if [ $retval != 0 ] 312 then 313 eval "$SSH_CMD \"uname\" > $RUN_DIR/osync_remote_os_$SCRIPT_PID 2>&1" & 314 child_pid=$! 315 WaitForTaskCompletion $child_pid 120 240 316 retval=$? 317 if [ $retval != 0 ] 318 then 319 LogError "Cannot Get remote OS type." 320 fi 321 fi 322 fi 323 REMOTE_OS_VAR=$(cat $RUN_DIR/osync_remote_os_$SCRIPT_PID) 324 case $REMOTE_OS_VAR in 325 *"Linux"*) 326 REMOTE_OS="Linux" 327 ;; 328 *"BSD"*) 329 REMOTE_OS="BSD" 330 ;; 331 *"MINGW32"*) 332 REMOTE_OS="msys" 333 ;; 334 *"Darwin"*) 335 REMOTE_OS="MacOSX" 336 ;; 337 *"ssh"*|*"SSH"*) 338 LogError "Cannot connect to remote system." 339 exit 1 340 ;; 341 *) 342 LogError "Running on remote OS failed. Please report to the author if the OS is not supported." 343 LogError "Remote OS said:\n$REMOTE_OS_VAR" 344 exit 1 345 esac 346 LogDebug "Remote OS: [$REMOTE_OS_VAR]." 347 fi 348 } 349 # Waits for pid $1 to complete. Will log an alert if $2 seconds passed since current task execution unless $2 equals 0. 350 # Will stop task and log alert if $3 seconds passed since current task execution unless $3 equals 0. 351 function WaitForTaskCompletion 352 { 353 soft_alert=0 354 log_ttime=0 355 SECONDS_BEGIN=$SECONDS 356 while eval "$PROCESS_TEST_CMD" > /dev/null 357 do 358 Spinner 359 EXEC_TIME=$(($SECONDS - $SECONDS_BEGIN)) 360 if [ $((($EXEC_TIME + 1) % $KEEP_LOGGING)) -eq 0 ] 361 then 362 if [ $log_ttime -ne $EXEC_TIME ] 363 then 364 log_ttime=$EXEC_TIME 365 Log "Current task still running." 366 fi 367 fi 368 if [ $EXEC_TIME -gt "$2" ] 369 then 370 if [ $soft_alert -eq 0 ] && [ "$2" != 0 ] 371 then 372 LogError "Max soft execution time exceeded for task." 373 soft_alert=1 374 fi 375 if [ $EXEC_TIME -gt "$3" ] && [ "$3" != 0 ] 376 then 377 LogError "Max hard execution time exceeded for task. Stopping task execution." 378 kill -s SIGTERM $1 379 if [ $? == 0 ] 380 then 381 LogError "Task stopped succesfully" 382 else 383 LogError "Sending SIGTERM to proces failed. Trying the hard way." 384 kill -9 $1 385 if [ $? != 0 ] 386 then 387 LogError "Could not stop task." 388 fi 389 fi 390 return 1 391 fi 392 fi 393 sleep $SLEEP_TIME 394 done 395 wait $child_pid 396 return $? 397 } 398 # Waits for pid $1 to complete. Will log an alert if $2 seconds passed since script start unless $2 equals 0. 399 # Will stop task and log alert if $3 seconds passed since script start unless $3 equals 0. 400 function WaitForCompletion 401 { 402 soft_alert=0 403 log_time=0 404 while eval "$PROCESS_TEST_CMD" > /dev/null 405 do 406 Spinner 407 if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ] 408 then 409 if [ $log_time -ne $EXEC_TIME ] 410 then 411 log_time=$EXEC_TIME 412 Log "Current task still running." 413 fi 414 fi 415 if [ $SECONDS -gt "$2" ] 416 then 417 if [ $soft_alert -eq 0 ] && [ "$2" != 0 ] 418 then 419 LogError "Max soft execution time exceeded for script." 420 soft_alert=1 421 fi 422 if [ $SECONDS -gt "$3" ] && [ "$3" != 0 ] 423 then 424 LogError "Max hard execution time exceeded for script. Stopping current task execution." 425 kill -s SIGTERM $1 426 if [ $? == 0 ] 427 then 428 LogError "Task stopped succesfully" 429 else 430 LogError "Sending SIGTERM to proces failed. Trying the hard way." 431 kill -9 $1 432 if [ $? != 0 ] 433 then 434 LogError "Could not stop task." 435 fi 436 fi 437 return 1 438 fi 439 fi 440 sleep $SLEEP_TIME 441 done 442 wait $child_pid 443 return $? 444 } 445 ## Runs local command $1 and waits for completition in $2 seconds 446 function RunLocalCommand 447 { 448 if [ $dryrun -ne 0 ] 449 then 450 Log "Dryrun: Local command [$1] not run." 451 return 1 452 fi 453 Log "Running command [$1] on local host." 454 eval "$1" > $RUN_DIR/osync_run_local_$SCRIPT_PID 2>&1 & 455 child_pid=$! 456 WaitForTaskCompletion $child_pid 0 $2 457 retval=$? 458 if [ $retval -eq 0 ] 459 then 460 Log "Command succeded." 461 else 462 LogError "Command failed." 463 fi 464 if [ $verbose -eq 1 ] || [ $retval -ne 0 ] 465 then 466 Log "Command output:\n$(cat $RUN_DIR/osync_run_local_$SCRIPT_PID)" 467 fi 468 if [ "$STOP_ON_CMD_ERROR" == "yes" ] && [ $retval -ne 0 ] 469 then 470 exit 1 471 fi 472 } 473 ## Runs remote command $1 and waits for completition in $2 seconds 474 function RunRemoteCommand 475 { 476 CheckConnectivity3rdPartyHosts 477 CheckConnectivityRemoteHost 478 if [ $dryrun -ne 0 ] 479 then 480 Log "Dryrun: Local command [$1] not run." 481 return 1 482 fi 483 Log "Running command [$1] on remote host." 484 eval "$SSH_CMD \"$1\" > $RUN_DIR/osync_run_remote_$SCRIPT_PID 2>&1 &" 485 child_pid=$! 486 WaitForTaskCompletion $child_pid 0 $2 487 retval=$? 488 if [ $retval -eq 0 ] 489 then 490 Log "Command succeded." 491 else 492 LogError "Command failed." 493 fi 494 if [ -f $RUN_DIR/osync_run_remote_$SCRIPT_PID ] && ([ $verbose -eq 1 ] || [ $retval -ne 0 ]) 495 then 496 Log "Command output:\n$(cat $RUN_DIR/osync_run_remote_$SCRIPT_PID)" 497 fi 498 if [ "$STOP_ON_CMD_ERROR" == "yes" ] && [ $retval -ne 0 ] 499 then 500 exit 1 501 fi 502 } 503 function RunBeforeHook 504 { 505 if [ "$LOCAL_RUN_BEFORE_CMD" != "" ] 506 then 507 RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE 508 fi 509 if [ "$REMOTE_RUN_BEFORE_CMD" != "" ] 510 then 511 RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE 512 fi 513 } 514 function RunAfterHook 515 { 516 if [ "$LOCAL_RUN_AFTER_CMD" != "" ] 517 then 518 RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER 519 fi 520 if [ "$REMOTE_RUN_AFTER_CMD" != "" ] 521 then 522 RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER 523 fi 524 } 525 function CheckConnectivityRemoteHost 526 { 527 if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_SYNC" != "no" ] 528 then 529 eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" 530 if [ $? != 0 ] 531 then 532 LogError "Cannot ping $REMOTE_HOST" 533 exit 1 534 fi 535 fi 536 } 537 function CheckConnectivity3rdPartyHosts 538 { 539 if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ] 540 then 541 remote_3rd_party_success=0 542 OLD_IFS=$IFS 543 IFS=$' \t\n' 544 for i in $REMOTE_3RD_PARTY_HOSTS 545 do 546 eval "$PING_CMD $i > /dev/null 2>&1" 547 if [ $? != 0 ] 548 then 549 Log "Cannot ping 3rd party host $i" 550 else 551 remote_3rd_party_success=1 552 fi 553 done 554 IFS=$OLD_IFS 555 if [ $remote_3rd_party_success -ne 1 ] 556 then 557 LogError "No remote 3rd party host responded to ping. No internet ?" 558 exit 1 559 fi 560 fi 561 } 562 ############################################################################################ 563 ### realpath.sh implementation from https://github.com/mkropat/sh-realpath 564 realpath() { 565 canonicalize_path "$(resolve_symlinks "$1")" 566 } 567 resolve_symlinks() { 568 _resolve_symlinks "$1" 569 } 570 _resolve_symlinks() { 571 _assert_no_path_cycles "$@" || return 572 local dir_context path 573 path=$(readlink -- "$1") 574 if [ $? -eq 0 ]; then 575 dir_context=$(dirname -- "$1") 576 _resolve_symlinks "$(_prepend_dir_context_if_necessary "$dir_context" "$path")" "$@" 577 else 578 printf '%s\n' "$1" 579 fi 580 } 581 _prepend_dir_context_if_necessary() { 582 if [ "$1" = . ]; then 583 printf '%s\n' "$2" 584 else 585 _prepend_path_if_relative "$1" "$2" 586 fi 587 } 588 _prepend_path_if_relative() { 589 case "$2" in 590 /* ) printf '%s\n' "$2" ;; 591 * ) printf '%s\n' "$1/$2" ;; 592 esac 593 } 594 _assert_no_path_cycles() { 595 local target path 596 target=$1 597 shift 598 for path in "$@"; do 599 if [ "$path" = "$target" ]; then 600 return 1 601 fi 602 done 603 } 604 canonicalize_path() { 605 if [ -d "$1" ]; then 606 _canonicalize_dir_path "$1" 607 else 608 _canonicalize_file_path "$1" 609 fi 610 } 611 _canonicalize_dir_path() { 612 (cd "$1" 2>/dev/null && pwd -P) 613 } 614 _canonicalize_file_path() { 615 local dir file 616 dir=$(dirname -- "$1") 617 file=$(basename -- "$1") 618 (cd "$dir" 2>/dev/null && printf '%s/%s\n' "$(pwd -P)" "$file") 619 } 620 # Optionally, you may also want to include: 621 ### readlink emulation ### 622 readlink() { 623 if _has_command readlink; then 624 _system_readlink "$@" 625 else 626 _emulated_readlink "$@" 627 fi 628 } 629 _has_command() { 630 hash -- "$1" 2>/dev/null 631 } 632 _system_readlink() { 633 command readlink "$@" 634 } 635 _emulated_readlink() { 636 if [ "$1" = -- ]; then 637 shift 638 fi 639 _gnu_stat_readlink "$@" || _bsd_stat_readlink "$@" 640 } 641 _gnu_stat_readlink() { 642 local output 643 output=$(stat -c %N -- "$1" 2>/dev/null) && 644 printf '%s\n' "$output" | 645 sed "s/^‘[^’]*’ -> ‘\(.*\)’/\1/ 646 s/^'[^']*' -> '\(.*\)'/\1/" 647 # FIXME: handle newlines 648 } 649 _bsd_stat_readlink() { 650 stat -f %Y -- "$1" 2>/dev/null 651 } 652 ### Specfic Osync function 653 function CreateOsyncDirs 654 { 655 if ! [ -d "$MASTER_STATE_DIR" ] 656 then 657 mkdir -p "$MASTER_STATE_DIR" 658 if [ $? != 0 ] 659 then 660 LogError "Cannot create master replica state dir [$MASTER_STATE_DIR]." 661 exit 1 662 fi 663 fi 664 if [ "$REMOTE_SYNC" == "yes" ] 665 then 666 CheckConnectivity3rdPartyHosts 667 CheckConnectivityRemoteHost 668 eval "$SSH_CMD \"if ! [ -d \\\"$SLAVE_STATE_DIR\\\" ]; then $COMMAND_SUDO mkdir -p \\\"$SLAVE_STATE_DIR\\\"; fi 2>&1\"" & 669 child_pid=$! 670 WaitForTaskCompletion $child_pid 0 1800 671 else 672 if ! [ -d "$SLAVE_STATE_DIR" ] 673 then 674 mkdir -p "$SLAVE_STATE_DIR" > $RUN_DIR/osync_createosyncdirs_$SCRIPT_PID 2>&1 675 fi 676 fi 677 if [ $? != 0 ] 678 then 679 LogError "Cannot create slave replica state dir [$SLAVE_STATE_DIR]." 680 LogErorr "Command output:\n$(cat $RUN_DIR/osync_createosyncdirs_$SCRIPT_PID)" 681 exit 1 682 fi 683 } 684 function CheckMasterSlaveDirs 685 { 686 MASTER_SYNC_DIR_CANN=$(realpath "$MASTER_SYNC_DIR") 687 SLAVE_SYNC_DIR_CANN=$(realpath "$SLAVE_SYNC_DIR") 688 if [ "$REMOTE_SYNC" != "yes" ] 689 then 690 if [ "$MASTER_SYNC_DIR_CANN" == "$SLAVE_SYNC_DIR_CANN" ] 691 then 692 LogError "Master directory [$MASTER_SYNC_DIR] can't be the same as slave directory." 693 exit 1 694 fi 695 fi 696 if ! [ -d "$MASTER_SYNC_DIR" ] 697 then 698 if [ "$CREATE_DIRS" == "yes" ] 699 then 700 mkdir -p "$MASTER_SYNC_DIR" > $RUN_DIR/osync_checkmasterslavedirs_$SCRIPT_PID 2>&1 701 if [ $? != 0 ] 702 then 703 LogError "Cannot create master directory [$MASTER_SYNC_DIR]." 704 LogError "Command output:\n$(cat $RUN_DIR/osync_checkmasterslavedirs_$SCRIPT_PID)" 705 exit 1 706 fi 707 else 708 LogError "Master directory [$MASTER_SYNC_DIR] does not exist." 709 exit 1 710 fi 711 fi 712 if [ "$REMOTE_SYNC" == "yes" ] 713 then 714 CheckConnectivity3rdPartyHosts 715 CheckConnectivityRemoteHost 716 if [ "$CREATE_DIRS" == "yes" ] 717 then 718 eval "$SSH_CMD \"if ! [ -d \\\"$SLAVE_SYNC_DIR\\\" ]; then $COMMAND_SUDO mkdir -p \\\"$SLAVE_SYNC_DIR\\\"; fi 2>&1"\" > $RUN_DIR/osync_checkmasterslavedirs_$SCRIPT_PID & 719 child_pid=$! 720 WaitForTaskCompletion $child_pid 0 1800 721 if [ $? != 0 ] 722 then 723 LogError "Cannot create slave directory [$SLAVE_SYNC_DIR]." 724 LogError "Command output:\n$(cat $RUN_DIR/osync_checkmasterslavedirs_$SCRIPT_PID)" 725 exit 1 726 fi 727 else 728 eval "$SSH_CMD \"if ! [ -d \\\"$SLAVE_SYNC_DIR\\\" ]; then exit 1; fi"\" & 729 child_pid=$! 730 WaitForTaskCompletion $child_pid 0 1800 731 res=$? 732 if [ $res != 0 ] 733 then 734 LogError "Slave directory [$SLAVE_SYNC_DIR] does not exist." 735 exit 1 736 fi 737 fi 738 else 739 if [ ! -d "$SLAVE_SYNC_DIR" ] 740 then 741 if [ "$CREATE_DIRS" == "yes" ] 742 then 743 mkdir -p "$SLAVE_SYNC_DIR" 744 if [ $? != 0 ] 745 then 746 LogError "Cannot create slave directory [$SLAVE_SYNC_DIR]." 747 exit 1 748 else 749 Log "Created slave directory [$SLAVE_SYNC_DIR]." 750 fi 751 else 752 LogError "Slave directory [$SLAVE_SYNC_DIR] does not exist." 753 exit 1 754 fi 755 fi 756 fi 757 } 758 function CheckMinimumSpace 759 { 760 Log "Checking minimum disk space on master and slave." 761 MASTER_SPACE=$(df -P "$MASTER_SYNC_DIR" | tail -1 | awk '{print $4}') 762 if [ $MASTER_SPACE -lt $MINIMUM_SPACE ] 763 then 764 LogError "There is not enough free space on master [$MASTER_SPACE KB]." 765 fi 766 if [ "$REMOTE_SYNC" == "yes" ] 767 then 768 CheckConnectivity3rdPartyHosts 769 CheckConnectivityRemoteHost 770 eval "$SSH_CMD \"$COMMAND_SUDO df -P \\\"$SLAVE_SYNC_DIR\\\"\"" > $RUN_DIR/osync_slave_space_$SCRIPT_PID & 771 child_pid=$! 772 WaitForTaskCompletion $child_pid 0 1800 773 SLAVE_SPACE=$(cat $RUN_DIR/osync_slave_space_$SCRIPT_PID | tail -1 | awk '{print $4}') 774 else 775 SLAVE_SPACE=$(df -P "$SLAVE_SYNC_DIR" | tail -1 | awk '{print $4}') 776 fi 777 if [ $SLAVE_SPACE -lt $MINIMUM_SPACE ] 778 then 779 LogError "There is not enough free space on slave [$SLAVE_SPACE KB]." 780 fi 781 } 782 function RsyncExcludePattern 783 { 784 # Disable globbing so wildcards from exclusions don't get expanded 785 set -f 786 rest="$RSYNC_EXCLUDE_PATTERN" 787 while [ -n "$rest" ] 788 do 789 # Take the string until first occurence until $PATH_SEPARATOR_CHAR 790 str=${rest%%;*} 791 # Handle the last case 792 if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ] 793 then 794 rest= 795 else 796 # Cut everything before the first occurence of $PATH_SEPARATOR_CHAR 797 rest=${rest#*$PATH_SEPARATOR_CHAR} 798 fi 799 if [ "$RSYNC_EXCLUDE" == "" ] 800 then 801 RSYNC_EXCLUDE="--exclude=\"$str\"" 802 else 803 RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$str\"" 804 fi 805 done 806 set +f 807 } 808 function RsyncExcludeFrom 809 { 810 if [ ! "$RSYNC_EXCLUDE_FROM" == "" ] 811 then 812 ## Check if the exclude list has a full path, and if not, add the config file path if there is one 813 if [ "$(basename $RSYNC_EXCLUDE_FROM)" == "$RSYNC_EXCLUDE_FROM" ] 814 then 815 RSYNC_EXCLUDE_FROM=$(dirname $ConfigFile)/$RSYNC_EXCLUDE_FROM 816 fi 817 if [ -e "$RSYNC_EXCLUDE_FROM" ] 818 then 819 RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\"" 820 fi 821 fi 822 } 823 function WriteLockFiles 824 { 825 echo $SCRIPT_PID > "$MASTER_LOCK" 826 if [ $? != 0 ] 827 then 828 LogError "Could not set lock on master replica." 829 exit 1 830 else 831 Log "Locked master replica." 832 fi 833 if [ "$REMOTE_SYNC" == "yes" ] 834 then 835 CheckConnectivity3rdPartyHosts 836 CheckConnectivityRemoteHost 837 eval "$SSH_CMD \"echo $SCRIPT_PID@$SYNC_ID | $COMMAND_SUDO tee \\\"$SLAVE_LOCK\\\" > /dev/null \"" & 838 child_pid=$! 839 WaitForTaskCompletion $child_pid 0 1800 840 if [ $? != 0 ] 841 then 842 LogError "Could not set lock on remote slave replica." 843 exit 1 844 else 845 Log "Locked remote slave replica." 846 fi 847 else 848 echo "$SCRIPT_PID@$SYNC_ID" > "$SLAVE_LOCK" 849 if [ $? != 0 ] 850 then 851 LogError "Couuld not set lock on local slave replica." 852 exit 1 853 else 854 Log "Locked local slave replica." 855 fi 856 fi 857 } 858 function LockDirectories 859 { 860 if [ $nolocks -eq 1 ] 861 then 862 return 0 863 fi 864 if [ $force_unlock -eq 1 ] 865 then 866 WriteLockFiles 867 if [ $? != 0 ] 868 then 869 exit 1 870 fi 871 fi 872 Log "Checking for replica locks." 873 if [ -f "$MASTER_LOCK" ] 874 then 875 master_lock_pid=$(cat $MASTER_LOCK) 876 LogDebug "Master lock pid present: $master_lock_pid" 877 ps -p$master_lock_pid > /dev/null 2>&1 878 if [ $? != 0 ] 879 then 880 Log "There is a dead osync lock on master. Instance $master_lock_pid no longer running. Resuming." 881 else 882 LogError "There is already a local instance of osync that locks master replica. Cannot start. If your are sure this is an error, plaese kill instance $master_lock_pid of osync." 883 exit 1 884 fi 885 fi 886 if [ "$REMOTE_SYNC" == "yes" ] 887 then 888 CheckConnectivity3rdPartyHosts 889 CheckConnectivityRemoteHost 890 eval "$SSH_CMD \"if [ -f \\\"$SLAVE_LOCK\\\" ]; then cat \\\"$SLAVE_LOCK\\\"; fi\" > $RUN_DIR/osync_remote_slave_lock_$SCRIPT_PID" & 891 child_pid=$! 892 WaitForTaskCompletion $child_pid 0 1800 893 if [ -f $RUN_DIR/osync_remote_slave_lock_$SCRIPT_PID ] 894 then 895 slave_lock_pid=$(cat $RUN_DIR/osync_remote_slave_lock_$SCRIPT_PID | cut -d'@' -f1) 896 slave_lock_id=$(cat $RUN_DIR/osync_remote_slave_lock_$SCRIPT_PID | cut -d'@' -f2) 897 fi 898 else 899 if [ -f "$SLAVE_LOCK" ] 900 then 901 slave_lock_pid=$(cat "$SLAVE_LOCK" | cut -d'@' -f1) 902 slave_lock_id=$(cat "$SLAVE_LOCK" | cut -d'@' -f2) 903 fi 904 fi 905 if [ "$slave_lock_pid" != "" ] && [ "$slave_lock_id" != "" ] 906 then 907 LogDebug "Slave lock pid: $slave_lock_pid" 908 LogDebug "Slave lock id: $slave_lock_pid" 909 ps -p$slave_lock_pid > /dev/null 910 if [ $? != 0 ] 911 then 912 if [ "$slave_lock_id" == "$SYNC_ID" ] 913 then 914 Log "There is a dead osync lock on slave replica that corresponds to this master sync-id. Instance $slave_lock_pid no longer running. Resuming." 915 else 916 if [ "$FORCE_STRANGER_LOCK_RESUME" == "yes" ] 917 then 918 LogError "WARNING: There is a dead osync lock on slave replica that does not correspond to this master sync-id. Forcing resume." 919 else 920 LogError "There is a dead osync lock on slave replica that does not correspond to this master sync-id. Will not resume." 921 exit 1 922 fi 923 fi 924 else 925 LogError "There is already a local instance of osync that locks slave replica. Cannot start. If you are sure this is an error, please kill instance $slave_lock_pid of osync." 926 exit 1 927 fi 928 fi 929 WriteLockFiles 930 } 931 function UnlockDirectories 932 { 933 if [ $nolocks -eq 1 ] 934 then 935 return 0 936 fi 937 if [ "$REMOTE_SYNC" == "yes" ] 938 then 939 CheckConnectivity3rdPartyHosts 940 CheckConnectivityRemoteHost 941 eval "$SSH_CMD \"if [ -f \\\"$SLAVE_LOCK\\\" ]; then $COMMAND_SUDO rm \\\"$SLAVE_LOCK\\\"; fi 2>&1\"" > $RUN_DIR/osync_UnlockDirectories_$SCRIPT_PID & 942 child_pid=$! 943 WaitForTaskCompletion $child_pid 0 1800 944 else 945 if [ -f "$SLAVE_LOCK" ] 946 then 947 rm "$SLAVE_LOCK" > $RUN_DIR/osync_UnlockDirectories_$SCRIPT_PID 2>&1 948 fi 949 fi 950 if [ $? != 0 ] 951 then 952 LogError "Could not unlock slave replica." 953 LogError "Command Output:\n$(cat $RUN_DIR/osync_UnlockDirectories_$SCRIPT_PID)" 954 else 955 Log "Removed slave replica lock." 956 fi 957 if [ -f "$MASTER_LOCK" ] 958 then 959 rm "$MASTER_LOCK" 960 if [ $? != 0 ] 961 then 962 LogError "Could not unlock master replica." 963 else 964 Log "Removed master replica lock." 965 fi 966 fi 967 } 968 ###### Sync core functions 969 ## Rsync does not like spaces in directory names, considering it as two different directories. Handling this schema by escaping space. 970 ## It seems this only happens when trying to execute an rsync command through eval $rsync_cmd on a remote host. 971 ## So i'm using unescaped $MASTER_SYNC_DIR for local rsync calls and escaped $ESC_MASTER_SYNC_DIR for remote rsync calls like user@host:$ESC_MASTER_SYNC_DIR 972 ## The same applies for slave sync dir..............................................T.H.I.S..I.S..A..P.R.O.G.R.A.M.M.I.N.G..N.I.G.H.T.M.A.R.E 973 ## tree_list(replica_path, replica type, tree_filename) Creates a list of files in replica_path for replica type (master/slave) in filename $3 974 function tree_list 975 { 976 Log "Creating $2 replica file list [$1]." 977 if [ "$REMOTE_SYNC" == "yes" ] && [ "$2" == "slave" ] 978 then 979 CheckConnectivity3rdPartyHosts 980 CheckConnectivityRemoteHost 981 ESC=$(EscapeSpaces "$1") 982 rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$ESC/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/osync_$2_$SCRIPT_PID\" &" 983 else 984 rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --list-only \"$1/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/osync_$2_$SCRIPT_PID\" &" 985 fi 986 LogDebug "RSYNC_CMD: $rsync_cmd" 987 ## Redirect commands stderr here to get rsync stderr output in logfile 988 eval $rsync_cmd 2>> "$LOG_FILE" 989 child_pid=$! 990 WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME 991 retval=$? 992 ## Retval 24 = some files vanished while creating list 993 if ([ $retval == 0 ] || [ $retval == 24 ]) && [ -f $RUN_DIR/osync_$2_$SCRIPT_PID ] 994 then 995 mv $RUN_DIR/osync_$2_$SCRIPT_PID "$MASTER_STATE_DIR/$2$3" 996 return $? 997 else 998 LogError "Cannot create replica file list." 999 exit $retval 1000 fi 1001 } 1002 # delete_list(replica, tree-file-after, tree-file-current, deleted-list-file, deleted-failed-list-file): Creates a list of files vanished from last run on replica $1 (master/slave) 1003 function delete_list 1004 { 1005 Log "Creating $1 replica deleted file list." 1006 if [ -f "$MASTER_STATE_DIR/$1$TREE_AFTER_FILENAME_NO_SUFFIX" ] 1007 then 1008 ## Same functionnality, comm is much faster than grep but is not available on every platform 1009 if type -p comm > /dev/null 2>&1 1010 then 1011 cmd="comm -23 \"$MASTER_STATE_DIR/$1$TREE_AFTER_FILENAME_NO_SUFFIX\" \"$MASTER_STATE_DIR/$1$3\" > \"$MASTER_STATE_DIR/$1$4\"" 1012 else 1013 ## The || : forces the command to have a good result 1014 cmd="(grep -F -x -v -f \"$MASTER_STATE_DIR/$1$3\" \"$MASTER_STATE_DIR/$1$TREE_AFTER_FILENAME_NO_SUFFIX\" || :) > \"$MASTER_STATE_DIR/$1$4\"" 1015 fi 1016 LogDebug "CMD: $cmd" 1017 eval $cmd 1018 retval=$? 1019 # Add delete failed file list to current delete list and then empty it 1020 if [ -f "$MASTER_STATE_DIR/$1$5" ] 1021 then 1022 cat "$MASTER_STATE_DIR/$1$5" >> "$MASTER_STATE_DIR/$1$4" 1023 rm -f "$MASTER_STATE_DIR/$1$5" 1024 fi 1025 return $retval 1026 else 1027 touch "$MASTER_STATE_DIR/$1$4" 1028 return $retval 1029 fi 1030 } 1031 # sync_update(source replica, destination replica, delete_list_filename) 1032 function sync_update 1033 { 1034 Log "Updating $2 replica." 1035 if [ "$1" == "master" ] 1036 then 1037 SOURCE_DIR="$MASTER_SYNC_DIR" 1038 ESC_SOURCE_DIR=$(EscapeSpaces "$MASTER_SYNC_DIR") 1039 DEST_DIR="$SLAVE_SYNC_DIR" 1040 ESC_DEST_DIR=$(EscapeSpaces "$SLAVE_SYNC_DIR") 1041 BACKUP_DIR="$SLAVE_BACKUP" 1042 else 1043 SOURCE_DIR="$SLAVE_SYNC_DIR" 1044 ESC_SOURCE_DIR=$(EscapeSpaces "$SLAVE_SYNC_DIR") 1045 DEST_DIR="$MASTER_SYNC_DIR" 1046 ESC_DEST_DIR=$(EscapeSpaces "$MASTER_SYNC_DIR") 1047 BACKUP_DIR="$MASTER_BACKUP" 1048 fi 1049 if [ "$REMOTE_SYNC" == "yes" ] 1050 then 1051 CheckConnectivity3rdPartyHosts 1052 CheckConnectivityRemoteHost 1053 if [ "$1" == "master" ] 1054 then 1055 rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$1$3\" --exclude-from=\"$MASTER_STATE_DIR/$2$3\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" 1056 else 1057 rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$2$3\" --exclude-from=\"$MASTER_STATE_DIR/$1$3\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" 1058 fi 1059 else 1060 rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$1$3\" --exclude-from=\"$MASTER_STATE_DIR/$2$3\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" 1061 fi 1062 LogDebug "RSYNC_CMD: $rsync_cmd" 1063 eval "$rsync_cmd" 1064 child_pid=$! 1065 WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME 1066 retval=$? 1067 if [ $verbose -eq 1 ] && [ -f $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID ] 1068 then 1069 Log "List:\n$(cat $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID)" 1070 fi 1071 if [ $retval != 0 ] && [ $retval != 24 ] 1072 then 1073 LogError "Updating $2 replica failed. Stopping execution." 1074 if [ $verbose -eq 0 ] && [ -f $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID ] 1075 then 1076 LogError "Rsync output:\n$(cat $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID)" 1077 fi 1078 exit $retval 1079 else 1080 Log "Updating $2 replica succeded." 1081 return 0 1082 fi 1083 } 1084 # delete_local(replica dir, delete file list, delete dir, delete failed file) 1085 function _delete_local 1086 { 1087 ## On every run, check wheter the next item is already deleted because it's included in a directory already deleted 1088 previous_file="" 1089 OLD_IFS=$IFS 1090 IFS=$'\r\n' 1091 for files in $(cat "$MASTER_STATE_DIR/$2") 1092 do 1093 if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ] 1094 then 1095 if [ "$SOFT_DELETE" != "no" ] 1096 then 1097 if [ ! -d "$REPLICA_DIR$3" ] 1098 then 1099 mkdir -p "$REPLICA_DIR$3" 1100 if [ $? != 0 ] 1101 then 1102 LogError "Cannot create replica deletion directory." 1103 fi 1104 fi 1105 if [ $verbose -eq 1 ] 1106 then 1107 Log "Soft deleting $REPLICA_DIR$files" 1108 fi 1109 if [ $dryrun -ne 1 ] 1110 then 1111 if [ -e "$REPLICA_DIR$3/$files" ] 1112 then 1113 rm -rf "$REPLICA_DIR$3/$files" 1114 fi 1115 # In order to keep full path on soft deletion, create parent directories before move 1116 parentdir="$(dirname "$files")" 1117 if [ "$parentdir" != "." ] 1118 then 1119 mkdir --parents "$REPLICA_DIR$3/$parentdir" 1120 mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$3/$parentdir" 1121 else 1122 mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$3" 1123 fi 1124 if [ $? != 0 ] 1125 then 1126 LogError "Cannot move $REPLICA_DIR$files to deletion directory." 1127 echo "$files" >> "$MASTER_STATE_DIR/$4" 1128 fi 1129 fi 1130 else 1131 if [ $verbose -eq 1 ] 1132 then 1133 Log "Deleting $REPLICA_DIR$files" 1134 fi 1135 if [ $dryrun -ne 1 ] 1136 then 1137 rm -rf "$REPLICA_DIR$files" 1138 if [ $? != 0 ] 1139 then 1140 LogError "Cannot delete $REPLICA_DIR$files" 1141 echo "$files" >> "$MASTER_STATE_DIR/$4" 1142 fi 1143 fi 1144 fi 1145 previous_file="$files" 1146 fi 1147 done 1148 IFS=$OLD_IFS 1149 } 1150 # delete_remote(replica dir, delete file list, delete dir, delete fail file list) 1151 function _delete_remote 1152 { 1153 ## This is a special coded function. Need to redelcare local functions on remote host, passing all needed variables as escaped arguments to ssh command. 1154 ## Anything beetween << ENDSSH and ENDSSH will be executed remotely 1155 # Additionnaly, we need to copy the deletetion list to the remote state folder 1156 ESC_DEST_DIR="$(EscapeSpaces "$SLAVE_STATE_DIR")" 1157 rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" \"$MASTER_STATE_DIR/$2\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_remote_deletion_list_copy_$SCRIPT_PID 2>&1" 1158 LogDebug "RSYNC_CMD: $rsync_cmd" 1159 eval $rsync_cmd 2>> "$LOG_FILE" 1160 if [ $? != 0 ] 1161 then 1162 LogError "Cannot copy the deletion list to remote replica." 1163 if [ -f $RUN_DIR/osync_remote_deletion_list_copy_$SCRIPT_PID ] 1164 then 1165 LogError "$(cat $RUN_DIR/osync_remote_deletion_list_copy_$SCRIPT_PID)" 1166 fi 1167 exit 1 1168 fi 1169 $SSH_CMD error_alert=0 sync_on_changes=$sync_on_changes silent=$silent DEBUG=$DEBUG dryrun=$dryrun verbose=$verbose COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "$SLAVE_STATE_DIR/$2")" REPLICA_DIR="$(EscapeSpaces "$REPLICA_DIR")" DELETE_DIR="$(EscapeSpaces "$DELETE_DIR")" FAILED_DELETE_LIST="$(EscapeSpaces "$SLAVE_STATE_DIR/$4")" 'bash -s' << 'ENDSSH' > $RUN_DIR/osync_remote_deletion_$SCRIPT_PID 2>&1 & 1170 ## The following lines are executed remotely 1171 function Log 1172 { 1173 if [ $sync_on_changes -eq 1 ] 1174 then 1175 prefix="$(date) - " 1176 else 1177 prefix="R-TIME: $SECONDS - " 1178 fi 1179 if [ $silent -eq 0 ] 1180 then 1181 echo -e "$prefix$1" 1182 fi 1183 } 1184 function LogError 1185 { 1186 Log "$1" 1187 error_alert=1 1188 } 1189 ## Empty earlier failed delete list 1190 > "$FAILED_DELETE_LIST" 1191 ## On every run, check wheter the next item is already deleted because it's included in a directory already deleted 1192 previous_file="" 1193 OLD_IFS=$IFS 1194 IFS=$'\r\n' 1195 for files in $(cat "$FILE_LIST") 1196 do 1197 if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ] 1198 then 1199 if [ ! -d "$REPLICA_DIR$DELETE_DIR" ] 1200 then 1201 $COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR" 1202 if [ $? != 0 ] 1203 then 1204 LogError "Cannot create replica deletion directory." 1205 fi 1206 fi 1207 if [ "$SOFT_DELETE" != "no" ] 1208 then 1209 if [ $verbose -eq 1 ] 1210 then 1211 Log "Soft deleting $REPLICA_DIR$files" 1212 fi 1213 if [ $dryrun -ne 1 ] 1214 then 1215 if [ -e "$REPLICA_DIR$DELETE_DIR/$files" ] 1216 then 1217 $COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETE_DIR/$files" 1218 fi 1219 # In order to keep full path on soft deletion, create parent directories before move 1220 parentdir="$(dirname "$files")" 1221 if [ "$parentdir" != "." ] 1222 then 1223 $COMMAND_SUDO mkdir --parents "$REPLICA_DIR$DELETE_DIR/$parentdir" 1224 $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR/$parentdir" 1225 else 1226 $COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR" 1227 fi 1228 if [ $? != 0 ] 1229 then 1230 LogError "Cannot move $REPLICA_DIR$files to deletion directory." 1231 echo "$files" >> "$FAILED_DELETE_LIST" 1232 fi 1233 fi 1234 else 1235 if [ $verbose -eq 1 ] 1236 then 1237 Log "Deleting $REPLICA_DIR$files" 1238 fi 1239 if [ $dryrun -ne 1 ] 1240 then 1241 $COMMAND_SUDO rm -rf "$REPLICA_DIR$files" 1242 if [ $? != 0 ] 1243 then 1244 LogError "Cannot delete $REPLICA_DIR$files" 1245 echo "$files" >> "$SLAVE_STATE_DIR/$FAILED_DELETE_LIST" 1246 fi 1247 fi 1248 fi 1249 previous_file="$files" 1250 fi 1251 done 1252 IFS=$OLD_IFS 1253 ENDSSH 1254 ## Need to add a trivial sleep time to give ssh time to log to local file 1255 sleep 5 1256 ## Copy back the deleted failed file list 1257 ESC_SOURCE_FILE="$(EscapeSpaces "$SLAVE_STATE_DIR/$4")" 1258 rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_FILE\" \"$MASTER_STATE_DIR\" > $RUN_DIR/osync_remote_failed_deletion_list_copy_$SCRIPT_PID" 1259 LogDebug "RSYNC_CMD: $rsync_cmd" 1260 eval $rsync_cmd 2>> "$LOG_FILE" 1261 if [ $? != 0 ] 1262 then 1263 LogError "Cannot copy back the failed deletion list to master replica." 1264 if [ -f $RUN_DIR/osync_remote_failed_deletion_list_copy_$SCRIPT_PID ] 1265 then 1266 LogError "$(cat $RUN_DIR/osync_remote_failed_deletion_list_copy_$SCRIPT_PID)" 1267 fi 1268 exit 1 1269 fi 1270 exit $? 1271 } 1272 # delete_propagation(replica name, deleted_list_filename, deleted_failed_file_list) 1273 # replica name = "master" / "slave" 1274 function deletion_propagation 1275 { 1276 Log "Propagating deletions to $1 replica." 1277 if [ "$1" == "master" ] 1278 then 1279 REPLICA_DIR="$MASTER_SYNC_DIR" 1280 DELETE_DIR="$MASTER_DELETE_DIR" 1281 _delete_local "$REPLICA_DIR" "slave$2" "$DELETE_DIR" "slave$3" & 1282 child_pid=$! 1283 WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME 1284 retval=$? 1285 if [ $retval != 0 ] 1286 then 1287 LogError "Deletion on replica $1 failed." 1288 exit 1 1289 fi 1290 else 1291 REPLICA_DIR="$SLAVE_SYNC_DIR" 1292 DELETE_DIR="$SLAVE_DELETE_DIR" 1293 if [ "$REMOTE_SYNC" == "yes" ] 1294 then 1295 _delete_remote "$REPLICA_DIR" "master$2" "$DELETE_DIR" "master$3" & 1296 else 1297 _delete_local "$REPLICA_DIR" "master$2" "$DELETE_DIR" "master$3" & 1298 fi 1299 child_pid=$! 1300 WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME 1301 retval=$? 1302 if [ $retval == 0 ] 1303 then 1304 if [ -f $RUN_DIR/osync_remote_deletion_$SCRIPT_PID ] && [ $verbose -eq 1 ] 1305 then 1306 Log "Remote:\n$(cat $RUN_DIR/osync_remote_deletion_$SCRIPT_PID)" 1307 fi 1308 return $retval 1309 else 1310 LogError "Deletion on remote system failed." 1311 if [ -f $RUN_DIR/osync_remote_deletion_$SCRIPT_PID ] 1312 then 1313 LogError "Remote:\n$(cat $RUN_DIR/osync_remote_deletion_$SCRIPT_PID)" 1314 fi 1315 exit 1 1316 fi 1317 fi 1318 } 1319 ###### Sync function in 5 steps of each 2 runs (functions above) 1320 ###### 1321 ###### Step 1: Create current tree list for master and slave replicas (Steps 1M and 1S) 1322 ###### Step 2: Create deleted file list for master and slave replicas (Steps 2M and 2S) 1323 ###### Step 3: Update master and slave replicas (Steps 3M and 3S, order depending on conflict prevalence) 1324 ###### Step 4: Deleted file propagation to master and slave replicas (Steps 4M and 4S) 1325 ###### Step 5: Create after run tree list for master and slave replicas (Steps 5M and 5S) 1326 function Sync 1327 { 1328 Log "Starting synchronization task." 1329 CheckConnectivity3rdPartyHosts 1330 CheckConnectivityRemoteHost 1331 if [ -f "$MASTER_LAST_ACTION" ] && [ "$RESUME_SYNC" != "no" ] 1332 then 1333 resume_sync=$(cat "$MASTER_LAST_ACTION") 1334 if [ -f "$MASTER_RESUME_COUNT" ] 1335 then 1336 resume_count=$(cat "$MASTER_RESUME_COUNT") 1337 else 1338 resume_count=0 1339 fi 1340 if [ $resume_count -lt $RESUME_TRY ] 1341 then 1342 if [ "$resume_sync" != "sync.success" ] 1343 then 1344 Log "WARNING: Trying to resume aborted osync execution on $($STAT_CMD "$MASTER_LAST_ACTION") at task [$resume_sync]. [$resume_count] previous tries." 1345 echo $(($resume_count+1)) > "$MASTER_RESUME_COUNT" 1346 else 1347 resume_sync=none 1348 fi 1349 else 1350 Log "Will not resume aborted osync execution. Too much resume tries [$resume_count]." 1351 echo "noresume" > "$MASTER_LAST_ACTION" 1352 echo "0" > "$MASTER_RESUME_COUNT" 1353 resume_sync=none 1354 fi 1355 else 1356 resume_sync=none 1357 fi 1358 ################################################################################################################################################# Actual sync begins here 1359 ## This replaces the case statement because ;& operator is not supported in bash 3.2... Code is more messy than case :( 1360 if [ "$resume_sync" == "none" ] || [ "$resume_sync" == "noresume" ] || [ "$resume_sync" == "master-replica-tree.fail" ] 1361 then 1362 #master_tree_current 1363 tree_list "$MASTER_SYNC_DIR" master "$TREE_CURRENT_FILENAME" 1364 if [ $? == 0 ] 1365 then 1366 echo "${SYNC_ACTION[0]}.success" > "$MASTER_LAST_ACTION" 1367 else 1368 echo "${SYNC_ACTION[0]}.fail" > "$MASTER_LAST_ACTION" 1369 fi 1370 resume_sync="resumed" 1371 fi 1372 if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[0]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[1]}.fail" ] 1373 then 1374 #slave_tree_current 1375 tree_list "$SLAVE_SYNC_DIR" slave "$TREE_CURRENT_FILENAME" 1376 if [ $? == 0 ] 1377 then 1378 echo "${SYNC_ACTION[1]}.success" > "$MASTER_LAST_ACTION" 1379 else 1380 echo "${SYNC_ACTION[1]}.fail" > "$MASTER_LAST_ACTION" 1381 fi 1382 resume_sync="resumed" 1383 fi 1384 if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[1]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[2]}.fail" ] 1385 then 1386 delete_list master "$TREE_AFTER_FILENAME" "$TREE_CURRENT_FILENAME" "$DELETED_LIST_FILENAME" "$FAILED_DELETE_LIST_FILENAME" 1387 if [ $? == 0 ] 1388 then 1389 echo "${SYNC_ACTION[2]}.success" > "$MASTER_LAST_ACTION" 1390 else 1391 echo "${SYNc_ACTION[2]}.fail" > "$MASTER_LAST_ACTION" 1392 fi 1393 resume_sync="resumed" 1394 fi 1395 if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[2]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.fail" ] 1396 then 1397 delete_list slave "$TREE_AFTER_FILENAME" "$TREE_CURRENT_FILENAME" "$DELETED_LIST_FILENAME" "$FAILED_DELETE_LIST_FILENAME" 1398 if [ $? == 0 ] 1399 then 1400 echo "${SYNC_ACTION[3]}.success" > "$MASTER_LAST_ACTION" 1401 else 1402 echo "${SYNC_ACTION[3]}.fail" > "$MASTER_LAST_ACTION" 1403 fi 1404 resume_sync="resumed" 1405 fi 1406 if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] 1407 then 1408 if [ "$CONFLICT_PREVALANCE" != "master" ] 1409 then 1410 if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ] 1411 then 1412 sync_update slave master "$DELETED_LIST_FILENAME" 1413 if [ $? == 0 ] 1414 then 1415 echo "${SYNC_ACTION[4]}.success" > "$MASTER_LAST_ACTION" 1416 else 1417 echo "${SYNC_ACTION[4]}.fail" > "$MASTER_LAST_ACTION" 1418 fi 1419 resume_sync="resumed" 1420 fi 1421 if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] 1422 then 1423 sync_update master slave "$DELETED_LIST_FILENAME" 1424 if [ $? == 0 ] 1425 then 1426 echo "${SYNC_ACTION[5]}.success" > "$MASTER_LAST_ACTION" 1427 else 1428 echo "${SYNC_ACTION[5]}.fail" > "$MASTER_LAST_ACTION" 1429 fi 1430 resume_sync="resumed" 1431 fi 1432 else 1433 if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] 1434 then 1435 sync_update master slave "$DELETED_LIST_FILENAME" 1436 if [ $? == 0 ] 1437 then 1438 echo "${SYNC_ACTION[5]}.success" > "$MASTER_LAST_ACTION" 1439 else 1440 echo "${SYNC_ACTION[5]}.fail" > "$MASTER_LAST_ACTION" 1441 fi 1442 resume_sync="resumed" 1443 fi 1444 if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ] 1445 then 1446 sync_update slave master "$DELETED_LIST_FILENAME" 1447 if [ $? == 0 ] 1448 then 1449 echo "${SYNC_ACTION[4]}.success" > "$MASTER_LAST_ACTION" 1450 else 1451 echo "${SYNC_ACTION[4]}.fail" > "$MASTER_LAST_ACTION" 1452 fi 1453 resume_sync="resumed" 1454 fi 1455 fi 1456 fi 1457 if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ] 1458 then 1459 deletion_propagation slave "$DELETED_LIST_FILENAME" "$FAILED_DELETE_LIST_FILENAME" 1460 if [ $? == 0 ] 1461 then 1462 echo "${SYNC_ACTION[6]}.success" > "$MASTER_LAST_ACTION" 1463 else 1464 echo "${SYNC_ACTION[6]}.fail" > "$MASTER_LAST_ACTION" 1465 fi 1466 resume_sync="resumed" 1467 fi 1468 if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.fail" ] 1469 then 1470 deletion_propagation master "$DELETED_LIST_FILENAME" "$FAILED_DELETE_LIST_FILENAME" 1471 if [ $? == 0 ] 1472 then 1473 echo "${SYNC_ACTION[7]}.success" > "$MASTER_LAST_ACTION" 1474 else 1475 echo "${SYNC_ACTION[7]}.fail" > "$MASTER_LAST_ACTION" 1476 fi 1477 resume_sync="resumed" 1478 fi 1479 if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.fail" ] 1480 then 1481 #master_tree_after 1482 tree_list "$MASTER_SYNC_DIR" master "$TREE_AFTER_FILENAME" 1483 if [ $? == 0 ] 1484 then 1485 echo "${SYNC_ACTION[8]}.success" > "$MASTER_LAST_ACTION" 1486 else 1487 echo "${SYNC_ACTION[8]}.fail" > "$MASTER_LAST_ACTION" 1488 fi 1489 resume_sync="resumed" 1490 fi 1491 if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[9]}.fail" ] 1492 then 1493 #slave_tree_after 1494 tree_list "$SLAVE_SYNC_DIR" slave "$TREE_AFTER_FILENAME" 1495 if [ $? == 0 ] 1496 then 1497 echo "${SYNC_ACTION[9]}.success" > "$MASTER_LAST_ACTION" 1498 else 1499 echo "${SYNC_ACTION[9]}.fail" > "$MASTER_LAST_ACTION" 1500 fi 1501 resume_sync="resumed" 1502 fi 1503 Log "Finished synchronization task." 1504 echo "${SYNC_ACTION[10]}" > "$MASTER_LAST_ACTION" 1505 echo "0" > "$MASTER_RESUME_COUNT" 1506 } 1507 function SoftDelete 1508 { 1509 if [ "$CONFLICT_BACKUP" != "no" ] && [ $CONFLICT_BACKUP_DAYS -ne 0 ] 1510 then 1511 Log "Running conflict backup cleanup." 1512 _SoftDelete $CONFLICT_BACKUP_DAYS "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR" "$SLAVE_SYNC_DIR$SLAVE_BACKUP_DIR" 1513 fi 1514 if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ] 1515 then 1516 Log "Running soft deletion cleanup." 1517 _SoftDelete $SOFT_DELETE_DAYS "$MASTER_SYNC_DIR$MASTER_DELETE_DIR" "$SLAVE_SYNC_DIR$SLAVE_DELETE_DIR" 1518 fi 1519 } 1520 # Takes 3 arguments 1521 # $1 = ctime (CONFLICT_BACKUP_DAYS or SOFT_DELETE_DAYS), $2 = MASTER_(BACKUP/DELETED)_DIR, $3 = SLAVE_(BACKUP/DELETED)_DIR 1522 function _SoftDelete 1523 { 1524 if [ -d "$2" ] 1525 then 1526 if [ $dryrun -eq 1 ] 1527 then 1528 Log "Listing files older than $1 days on master replica. Won't remove anything." 1529 else 1530 Log "Removing files older than $1 days on master replica." 1531 fi 1532 if [ $verbose -eq 1 ] 1533 then 1534 # Cannot launch log function from xargs, ugly hack 1535 $FIND_CMD "$2/" -type f -ctime +$1 -print0 | xargs -0 -I {} echo "Will delete file {}" > $RUN_DIR/osync_soft_delete_master_$SCRIPT_PID 1536 Log "Command output:\n$(cat $RUN_DIR/osync_soft_delete_master_$SCRIPT_PID)" 1537 $FIND_CMD "$2/" -type d -empty -ctime +$1 -print0 | xargs -0 -I {} echo "Will delete directory {}" > $RUN_DIR/osync_soft_delete_master_$SCRIPT_PID 1538 Log "Command output:\n$(cat $RUN_DIR/osync_soft_delete_master_$SCRIPT_PID)" 1539 fi 1540 if [ $dryrun -ne 1 ] 1541 then 1542 $FIND_CMD "$2/" -type f -ctime +$1 -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$2/" -type d -empty -ctime +$1 -print0 | xargs -0 -I {} rm -rf "{}" > $RUN_DIR/osync_soft_delete_master_$SCRIPT_PID 2>&1 & 1543 else 1544 Dummy & 1545 fi 1546 child_pid=$! 1547 WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME 1548 retval=$? 1549 if [ $retval -ne 0 ] 1550 then 1551 LogError "Error while executing cleanup on master replica." 1552 Log "Command output:\n$(cat $RUN_DIR/osync_soft_delete_master_$SCRIPT_PID)" 1553 else 1554 Log "Cleanup complete on master replica." 1555 fi 1556 elif [ -d "$2" ] && ! [ -w "$2" ] 1557 then 1558 LogError "Warning: Master replica dir [$2] isn't writable. Cannot clean old files." 1559 fi 1560 if [ "$REMOTE_SYNC" == "yes" ] 1561 then 1562 CheckConnectivity3rdPartyHosts 1563 CheckConnectivityRemoteHost 1564 if [ $dryrun -eq 1 ] 1565 then 1566 Log "Listing files older than $1 days on slave replica. Won't remove anything." 1567 else 1568 Log "Removing files older than $1 days on slave replica." 1569 fi 1570 if [ $verbose -eq 1 ] 1571 then 1572 # Cannot launch log function from xargs, ugly hack 1573 eval "$SSH_CMD \"if [ -w \\\"$3\\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\"$3/\\\" -type f -ctime +$1 -print0 | xargs -0 -I {} echo Will delete file {} && $REMOTE_FIND_CMD \\\"$3/\\\" -type d -empty -ctime $1 -print0 | xargs -0 -I {} echo Will delete directory {}; fi\"" > $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID 1574 Log "Command output:\n$(cat $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID)" 1575 fi 1576 if [ $dryrun -ne 1 ] 1577 then 1578 eval "$SSH_CMD \"if [ -w \\\"$3\\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\"$3/\\\" -type f -ctime +$1 -print0 | xargs -0 -I {} rm -f \\\"{}\\\" && $REMOTE_FIND_CMD \\\"$3/\\\" -type d -empty -ctime $1 -print0 | xargs -0 -I {} rm -rf \\\"{}\\\"; fi 2>&1\"" > $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID & 1579 else 1580 Dummy & 1581 fi 1582 child_pid=$! 1583 WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME 1584 retval=$? 1585 if [ $retval -ne 0 ] 1586 then 1587 LogError "Error while executing cleanup on slave replica." 1588 Log "Command output:\n$(cat $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID)" 1589 else 1590 Log "Cleanup complete on slave replica." 1591 fi 1592 else 1593 if [ -w "$3" ] 1594 then 1595 if [ $dryrun -eq 1 ] 1596 then 1597 Log "Listing files older than $1 days on slave replica. Won't remove anything." 1598 else 1599 Log "Removing files older than $1 days on slave replica." 1600 fi 1601 if [ $verbose -eq 1 ] 1602 then 1603 # Cannot launch log function from xargs, ugly hack 1604 $FIND_CMD "$3/" -type f -ctime +$1 -print0 | xargs -0 -I {} echo "Will delete file {}" > $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID 1605 Log "Command output:\n$(cat $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID)" 1606 $FIND_CMD "$3/" -type d -empty -ctime +$1 -print0 | xargs -0 -I {} echo "Will delete directory {}" > $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID 1607 Log "Command output:\n$(cat $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID)" 1608 Dummy & 1609 fi 1610 if [ $dryrun -ne 1 ] 1611 then 1612 $FIND_CMD "$3/" -type f -ctime +$1 -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$3/" -type d -empty -ctime +$1 -print0 | xargs -0 -I {} rm -rf "{}" > $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID & 1613 fi 1614 child_pid=$! 1615 WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME 1616 retval=$? 1617 if [ $retval -ne 0 ] 1618 then 1619 LogError "Error while executing cleanup on slave replica." 1620 Log "Command output:\n$(cat $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID)" 1621 else 1622 Log "Cleanup complete on slave replica." 1623 fi 1624 elif [ -d "$3" ] && ! [ -w "$3" ] 1625 then 1626 LogError "Warning: Slave replica dir [$3] isn't writable. Cannot clean old files." 1627 fi 1628 fi 1629 1630 } 1631 function Init 1632 { 1633 # Set error exit code if a piped command fails 1634 set -o pipefail 1635 set -o errtrace 1636 # Do not use exit and quit traps if osync runs in monitor mode 1637 if [ $sync_on_changes -eq 0 ] 1638 then 1639 trap TrapStop SIGINT SIGKILL SIGHUP SIGTERM SIGQUIT 1640 trap TrapQuit SIGKILL EXIT 1641 else 1642 trap TrapQuit SIGTERM EXIT SIGKILL SIGHUP SIGQUIT 1643 fi 1644 if [ "$DEBUG" == "yes" ] 1645 then 1646 trap 'TrapError ${LINENO} $?' ERR 1647 fi 1648 MAIL_ALERT_MSG="Warning: Execution of osync instance $OSYNC_ID (pid $SCRIPT_PID) as $LOCAL_USER@$LOCAL_HOST produced errors on $(date)." 1649 ## Test if slave dir is a ssh uri, and if yes, break it down it its values 1650 if [ "${SLAVE_SYNC_DIR:0:6}" == "ssh://" ] 1651 then 1652 REMOTE_SYNC="yes" 1653 # remove leadng 'ssh://' 1654 uri=${SLAVE_SYNC_DIR#ssh://*} 1655 if [[ "$uri" == *"@"* ]] 1656 then 1657 # remove everything after '@' 1658 REMOTE_USER=${uri%@*} 1659 else 1660 REMOTE_USER=$LOCAL_USER 1661 fi 1662 if [ "$SSH_RSA_PRIVATE_KEY" == "" ] 1663 then 1664 SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa 1665 fi 1666 # remove everything before '@' 1667 _hosturiandpath=${uri#*@} 1668 # remove everything after first '/' 1669 _hosturi=${_hosturiandpath%%/*} 1670 if [[ "$_hosturi" == *":"* ]] 1671 then 1672 REMOTE_PORT=${_hosturi##*:} 1673 else 1674 REMOTE_PORT=22 1675 fi 1676 REMOTE_HOST=${_hosturi%%:*} 1677 # remove everything before first '/' 1678 SLAVE_SYNC_DIR=${_hosturiandpath#*/} 1679 fi 1680 ## Make sure there is only one trailing slash on path 1681 MASTER_SYNC_DIR="${MASTER_SYNC_DIR%/}/" 1682 SLAVE_SYNC_DIR="${SLAVE_SYNC_DIR%/}/" 1683 MASTER_STATE_DIR="$MASTER_SYNC_DIR$OSYNC_DIR/state" 1684 SLAVE_STATE_DIR="$SLAVE_SYNC_DIR$OSYNC_DIR/state" 1685 STATE_DIR="$OSYNC_DIR/state" 1686 MASTER_LOCK="$MASTER_STATE_DIR/lock" 1687 SLAVE_LOCK="$SLAVE_STATE_DIR/lock" 1688 ## Working directories to keep backups of updated / deleted files 1689 MASTER_BACKUP_DIR="$OSYNC_DIR/backups" 1690 MASTER_DELETE_DIR="$OSYNC_DIR/deleted" 1691 SLAVE_BACKUP_DIR="$OSYNC_DIR/backups" 1692 SLAVE_DELETE_DIR="$OSYNC_DIR/deleted" 1693 ## Partial downloads dirs 1694 PARTIAL_DIR=$OSYNC_DIR"_partial" 1695 ## SSH compression 1696 if [ "$SSH_COMPRESSION" != "no" ] 1697 then 1698 SSH_COMP=-C 1699 else 1700 SSH_COMP= 1701 fi 1702 ## Define which runner (local bash or distant ssh) to use for standard commands and rsync commands 1703 if [ "$REMOTE_SYNC" == "yes" ] 1704 then 1705 SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" 1706 SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" 1707 RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -p $REMOTE_PORT" 1708 fi 1709 ## Support for older config files without RSYNC_EXECUTABLE option 1710 if [ "$RSYNC_EXECUTABLE" == "" ] 1711 then 1712 RSYNC_EXECUTABLE=rsync 1713 fi 1714 ## Sudo execution option 1715 if [ "$SUDO_EXEC" == "yes" ] 1716 then 1717 if [ "$RSYNC_REMOTE_PATH" != "" ] 1718 then 1719 RSYNC_PATH="sudo $RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" 1720 else 1721 RSYNC_PATH="sudo $RSYNC_EXECUTABLE" 1722 fi 1723 COMMAND_SUDO="sudo" 1724 else 1725 if [ "$RSYNC_REMOTE_PATH" != "" ] 1726 then 1727 RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" 1728 else 1729 RSYNC_PATH="$RSYNC_EXECUTABLE" 1730 fi 1731 COMMAND_SUDO="" 1732 fi 1733 ## Set rsync default arguments 1734 RSYNC_ARGS="-rlptgoD" 1735 if [ "$PRESERVE_ACL" == "yes" ] 1736 then 1737 RSYNC_ARGS=$RSYNC_ARGS" -A" 1738 fi 1739 if [ "$PRESERVE_XATTR" == "yes" ] 1740 then 1741 RSYNC_ARGS=$RSYNC_ARGS" -X" 1742 fi 1743 if [ "$RSYNC_COMPRESS" == "yes" ] 1744 then 1745 RSYNC_ARGS=$RSYNC_ARGS" -z" 1746 fi 1747 if [ "$COPY_SYMLINKS" == "yes" ] 1748 then 1749 RSYNC_ARGS=$RSYNC_ARGS" -L" 1750 fi 1751 if [ "$KEEP_DIRLINKS" == "yes" ] 1752 then 1753 RSYNC_ARGS=$RSYNC_ARGS" -K" 1754 fi 1755 if [ "$PRESERVE_HARDLINKS" == "yes" ] 1756 then 1757 RSYNC_ARGS=$RSYNC_ARGS" -H" 1758 fi 1759 if [ "$CHECKSUM" == "yes" ] 1760 then 1761 RSYNC_ARGS=$RSYNC_ARGS" --checksum" 1762 fi 1763 if [ $dryrun -eq 1 ] 1764 then 1765 RSYNC_ARGS=$RSYNC_ARGS" -n" 1766 DRY_WARNING="/!\ DRY RUN" 1767 fi 1768 if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ] 1769 then 1770 RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" 1771 fi 1772 ## Set sync only function arguments for rsync 1773 SYNC_OPTS="-u" 1774 if [ $verbose -eq 1 ] 1775 then 1776 SYNC_OPTS=$SYNC_OPTS"i" 1777 fi 1778 if [ $stats -eq 1 ] 1779 then 1780 SYNC_OPTS=$SYNC_OPTS" --stats" 1781 fi 1782 if [ "$PARTIAL" == "yes" ] 1783 then 1784 SYNC_OPTS=$SYNC_OPTS" --partial --partial-dir=\"$PARTIAL_DIR\"" 1785 RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$PARTIAL_DIR\"" 1786 fi 1787 ## Conflict options 1788 if [ "$CONFLICT_BACKUP" != "no" ] 1789 then 1790 MASTER_BACKUP="--backup --backup-dir=\"$MASTER_BACKUP_DIR\"" 1791 SLAVE_BACKUP="--backup --backup-dir=\"$SLAVE_BACKUP_DIR\"" 1792 if [ "$CONFLICT_BACKUP_MULTIPLE" == "yes" ] 1793 then 1794 MASTER_BACKUP="$MASTER_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)" 1795 SLAVE_BACKUP="$SLAVE_BACKUP --suffix .$(date +%Y.%m.%d-%H.%M.%S)" 1796 fi 1797 else 1798 MASTER_BACKUP= 1799 SLAVE_BACKUP= 1800 fi 1801 ## Add Rsync exclude patterns 1802 RsyncExcludePattern 1803 ## Add Rsync exclude from file 1804 RsyncExcludeFrom 1805 ## Filenames for state files 1806 if [ $dryrun -eq 1 ] 1807 then 1808 dry_suffix="-dry" 1809 fi 1810 TREE_CURRENT_FILENAME="-tree-current-$SYNC_ID$dry_suffix" 1811 TREE_AFTER_FILENAME="-tree-after-$SYNC_ID$dry_suffix" 1812 TREE_AFTER_FILENAME_NO_SUFFIX="-tree-after-$SYNC_ID" 1813 DELETED_LIST_FILENAME="-deleted-list-$SYNC_ID$dry_suffix" 1814 FAILED_DELETE_LIST_FILENAME="-failed-delete-$SYNC_ID$dry_suffix" 1815 MASTER_LAST_ACTION="$MASTER_STATE_DIR/last-action-$SYNC_ID$dry_suffix" 1816 MASTER_RESUME_COUNT="$MASTER_STATE_DIR/resume-count-$SYNC_ID$dry_suffix" 1817 ## Sync function actions (0-9) 1818 SYNC_ACTION=( 1819 'master-replica-tree' 1820 'slave-replica-tree' 1821 'master-deleted-list' 1822 'slave-deleted-list' 1823 'update-master-replica' 1824 'update-slave-replica' 1825 'delete-propagation-slave' 1826 'delete-propagation-master' 1827 'master-replica-tree-after' 1828 'slave-replica-tree-after' 1829 'sync.success' 1830 ) 1831 ## Set compression executable and extension 1832 COMPRESSION_LEVEL=3 1833 if type -p xz > /dev/null 2>&1 1834 then 1835 COMPRESSION_PROGRAM="| xz -$COMPRESSION_LEVEL" 1836 COMPRESSION_EXTENSION=.xz 1837 elif type -p lzma > /dev/null 2>&1 1838 then 1839 COMPRESSION_PROGRAM="| lzma -$COMPRESSION_LEVEL" 1840 COMPRESSION_EXTENSION=.lzma 1841 elif type -p pigz > /dev/null 2>&1 1842 then 1843 COMPRESSION_PROGRAM="| pigz -$COMPRESSION_LEVEL" 1844 COMPRESSION_EXTENSION=.gz 1845 COMPRESSION_OPTIONS=--rsyncable 1846 elif type -p gzip > /dev/null 2>&1 1847 then 1848 COMPRESSION_PROGRAM="| gzip -$COMPRESSION_LEVEL" 1849 COMPRESSION_EXTENSION=.gz 1850 COMPRESSION_OPTIONS=--rsyncable 1851 else 1852 COMPRESSION_PROGRAM= 1853 COMPRESSION_EXTENSION= 1854 fi 1855 ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" 1856 } 1857 function InitLocalOSSettings 1858 { 1859 ## If running under Msys, some commands don't run the same way 1860 ## Using mingw version of find instead of windows one 1861 ## Getting running processes is quite different 1862 ## Ping command isn't the same 1863 if [ "$LOCAL_OS" == "msys" ] 1864 then 1865 FIND_CMD=$(dirname $BASH)/find 1866 ## TODO: The following command needs to be checked on msys. Does the $1 variable substitution work ? 1867 PROCESS_TEST_CMD='ps -a | awk "{\$1=\$1}\$1" | awk "{print \$1}" | grep $1' 1868 PING_CMD="ping -n 2" 1869 else 1870 FIND_CMD=find 1871 PROCESS_TEST_CMD='ps -p$1' 1872 PING_CMD="ping -c 2 -i .2" 1873 fi 1874 ## Stat command has different syntax on Linux and FreeBSD/MacOSX 1875 if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ] 1876 then 1877 STAT_CMD="stat -f \"%Sm\"" 1878 else 1879 STAT_CMD="stat --format %y" 1880 fi 1881 } 1882 function InitRemoteOSSettings 1883 { 1884 ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) 1885 if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] 1886 then 1887 RSYNC_ARGS=$RSYNC_ARGS" -E" 1888 fi 1889 if [ "$REMOTE_OS" == "msys" ] 1890 then 1891 REMOTE_FIND_CMD=$(dirname $BASH)/find 1892 else 1893 REMOTE_FIND_CMD=find 1894 fi 1895 } 1896 function Main 1897 { 1898 CreateOsyncDirs 1899 LockDirectories 1900 Sync 1901 } 1902 function Usage 1903 { 1904 echo "$PROGRAM $PROGRAM_VERSION $PROGRAM_BUILD" 1905 echo $AUTHOR 1906 echo $CONTACT 1907 echo "" 1908 echo "You may use Osync with a full blown configuration file, or use its default options for quick command line sync." 1909 echo "Usage: osync.sh /path/to/config/file [OPTIONS]" 1910 echo "or osync.sh --master=/path/to/master/replica --slave=/path/to/slave/replica [OPTIONS] [QUICKSYNC OPTIONS]" 1911 echo "or osync.sh --master=/path/to/master/replica --slave=ssh://[backupuser]@remotehost.com[:portnumber]//path/to/slave/replica [OPTIONS] [QUICKSYNC OPTIONS]" 1912 echo "" 1913 echo "[OPTIONS]" 1914 echo "--dry Will run osync without actually doing anything; just testing" 1915 echo "--silent Will run osync without any output to stdout, used for cron jobs" 1916 echo "--verbose Increases output" 1917 echo "--stats Adds rsync transfer statistics to verbose output" 1918 echo "--partial Allows rsync to keep partial downloads that can be resumed later (experimental)" 1919 echo "--no-maxtime Disables any soft and hard execution time checks" 1920 echo "--force-unlock Will override any existing active or dead locks on master and slave replica" 1921 echo "--on-changes Will launch a sync task after a short wait period if there is some file activity on master replica. You should try daemon mode instead" 1922 echo "" 1923 echo "[QUICKSYNC OPTIONS]" 1924 echo "--master=\"\" Master replica path. Will contain state and backup directory (is mandatory)" 1925 echo "--slave=\"\" Local or remote slave replica path. Can be a ssh uri like ssh://user@host.com:22//path/to/slave/replica (is mandatory)" 1926 echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to slave replica" 1927 echo "--sync-id=\"\" Optional sync task name to identify this synchronization task when using multiple slaves" 1928 echo "" 1929 echo "Additionnaly, you may set most osync options at runtime. eg:" 1930 echo "SOFT_DELETE_DAYS=365 osync.sh --master=/path --slave=/other/path" 1931 echo "" 1932 exit 128 1933 } 1934 function SyncOnChanges 1935 { 1936 if ! type -p inotifywait > /dev/null 2>&1 1937 then 1938 LogError "No inotifywait command found. Cannot monitor changes." 1939 exit 1 1940 fi 1941 Log "#### Running Osync in file monitor mode." 1942 while true 1943 do 1944 if [ "$ConfigFile" != "" ] 1945 then 1946 cmd="bash $osync_cmd \"$ConfigFile\" $opts" 1947 else 1948 cmd="bash $osync_cmd $opts" 1949 fi 1950 eval $cmd 1951 retval=$? 1952 if [ $retval != 0 ] 1953 then 1954 LogError "osync child exited with error." 1955 exit $retval 1956 fi 1957 Log "#### Monitoring now." 1958 inotifywait --exclude $OSYNC_DIR $RSYNC_EXCLUDE -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$MASTER_SYNC_DIR" & 1959 sub_pid=$! 1960 wait $sub_pid 1961 retval=$? 1962 if [ $retval == 0 ] 1963 then 1964 Log "#### Changes detected, waiting $MIN_WAIT seconds before running next sync." 1965 sleep $MIN_WAIT 1966 elif [ $retval == 2 ] 1967 then 1968 Log "#### $MAX_WAIT timeout reached, running sync." 1969 else 1970 LogError "#### inotify error detected, waiting $MIN_WAIT seconds before running next sync." 1971 sleep $MIN_WAIT 1972 fi 1973 done 1974 } 1975 # Comand line argument flags 1976 dryrun=0 1977 silent=0 1978 if [ "$DEBUG" == "yes" ] 1979 then 1980 verbose=1 1981 else 1982 verbose=0 1983 fi 1984 stats=0 1985 PARTIAL=0 1986 force_unlock=0 1987 no_maxtime=0 1988 # Alert flags 1989 opts="" 1990 soft_alert_total=0 1991 error_alert=0 1992 soft_stop=0 1993 quick_sync=0 1994 sync_on_changes=0 1995 nolocks=0 1996 osync_cmd=$0 1997 if [ $# -eq 0 ] 1998 then 1999 Usage 2000 fi 2001 first=1 2002 for i in "$@" 2003 do 2004 case $i in 2005 --dry) 2006 dryrun=1 2007 opts=$opts" --dry" 2008 ;; 2009 --silent) 2010 silent=1 2011 opts=$opts" --silent" 2012 ;; 2013 --verbose) 2014 verbose=1 2015 opts=$opts" --verbose" 2016 ;; 2017 --stats) 2018 stats=1 2019 opts=$opts" --stats" 2020 ;; 2021 --partial) 2022 PARTIAL="yes" 2023 opts=$opts" --partial" 2024 ;; 2025 --force-unlock) 2026 force_unlock=1 2027 opts=$opts" --force-unlock" 2028 ;; 2029 --no-maxtime) 2030 no_maxtime=1 2031 opts=$opts" --no-maxtime" 2032 ;; 2033 --help|-h|--version|-v) 2034 Usage 2035 ;; 2036 --master=*) 2037 quick_sync=$(($quick_sync + 1)) 2038 no_maxtime=1 2039 MASTER_SYNC_DIR=${i##*=} 2040 opts=$opts" --master=\"$MASTER_SYNC_DIR\"" 2041 ;; 2042 --slave=*) 2043 quick_sync=$(($quick_sync + 1)) 2044 SLAVE_SYNC_DIR=${i##*=} 2045 opts=$opts" --slave=\"$SLAVE_SYNC_DIR\"" 2046 no_maxtime=1 2047 ;; 2048 --rsakey=*) 2049 SSH_RSA_PRIVATE_KEY=${i##*=} 2050 opts=$opts" --rsakey=\"$SSH_RSA_PRIVATE_KEY\"" 2051 ;; 2052 --sync-id=*) 2053 SYNC_ID=${i##*=} 2054 opts=$opts" --sync-id=\"$SYNC_ID\"" 2055 ;; 2056 --on-changes) 2057 sync_on_changes=1 2058 nolocks=1 2059 ;; 2060 --no-locks) 2061 nolocks=1 2062 ;; 2063 *) 2064 if [ $first == "0" ] 2065 then 2066 LogError "Unknown option '$i'" 2067 Usage 2068 fi 2069 ;; 2070 esac 2071 first=0 2072 done 2073 # Remove leading space if there is one 2074 opts="${opts# *}" 2075 CheckEnvironment 2076 if [ $? == 0 ] 2077 then 2078 ## Here we set default options for quicksync tasks when no configuration file is provided. 2079 if [ $quick_sync -eq 2 ] 2080 then 2081 if [ "$SYNC_ID" == "" ] 2082 then 2083 SYNC_ID="quicksync task" 2084 fi 2085 # Let the possibility to initialize those values directly via command line like SOFT_DELETE_DAYS=60 ./osync.sh 2086 if [ "$MINIMUM_SPACE" == "" ] 2087 then 2088 MINIMUM_SPACE=1024 2089 fi 2090 if [ "$CONFLICT_BACKUP_DAYS" == "" ] 2091 then 2092 CONFLICT_BACKUP_DAYS=30 2093 fi 2094 if [ "$SOFT_DELETE_DAYS" == "" ] 2095 then 2096 SOFT_DELETE_DAYS=30 2097 fi 2098 if [ "$RESUME_TRY" == "" ] 2099 then 2100 RESUME_TRY=1 2101 fi 2102 if [ "$SOFT_MAX_EXEC_TIME" == "" ] 2103 then 2104 SOFT_MAX_EXEC_TIME=0 2105 fi 2106 if [ "$HARD_MAX_EXEC_TIME" == "" ] 2107 then 2108 HARD_MAX_EXEC_TIME=0 2109 fi 2110 MIN_WAIT=30 2111 REMOTE_SYNC=no 2112 else 2113 ConfigFile="$1" 2114 LoadConfigFile "$ConfigFile" 2115 fi 2116 if [ "$LOGFILE" == "" ] 2117 then 2118 if [ -w /var/log ] 2119 then 2120 LOG_FILE=/var/log/osync_$SYNC_ID.log 2121 else 2122 LOG_FILE=./osync_$SYNC_ID.log 2123 fi 2124 else 2125 LOG_FILE="$LOGFILE" 2126 fi 2127 GetLocalOS 2128 InitLocalOSSettings 2129 Init 2130 GetRemoteOS 2131 InitRemoteOSSettings 2132 if [ $sync_on_changes -eq 1 ] 2133 then 2134 SyncOnChanges 2135 else 2136 DATE=$(date) 2137 Log "-------------------------------------------------------------" 2138 Log "$DRY_WARNING $DATE - $PROGRAM $PROGRAM_VERSION script begin." 2139 Log "-------------------------------------------------------------" 2140 Log "Sync task [$SYNC_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" 2141 if [ $no_maxtime -eq 1 ] 2142 then 2143 SOFT_MAX_EXEC_TIME=0 2144 HARD_MAX_EXEC_TIME=0 2145 fi 2146 CheckMasterSlaveDirs 2147 CheckMinimumSpace 2148 if [ $? == 0 ] 2149 then 2150 RunBeforeHook 2151 Main 2152 if [ $? == 0 ] 2153 then 2154 SoftDelete 2155 fi 2156 RunAfterHook 2157 fi 2158 fi 2159 else 2160 LogError "Environment not suitable to run osync." 2161 exit 1 2162 fi