#!/bin/bash # Copyright 2021 Luca Paris #This file is part of masync. #masync is free software: you can redistribute it and/or modify #it under the terms of the GNU General Public License as published by #the Free Software Foundation, either version 3 of the License, or #(at your option) any later version. #masync is distributed in the hope that it will be useful, #but WITHOUT ANY WARRANTY; without even the implied warranty of #MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #GNU General Public License for more details. #You should have received a copy of the GNU General Public License #along with masync. If not, see . source ~/bin/.synccmd.sh source ~/bin/.filetemplates.sh source ~/bin/.colordef.sh MYPID=$$ STATUS_RUNNING=RUNNING STATUS_STOPPED=STOPPED HELP_CMD_NAME='masync' HELP_LOCAL_DIR="/home/$USER/localsync/" HELP_REMOTE_DIR="remoteuser@server:/home/remoteuser/sync/" myhelp() { echo "Usage: ${HELP_CMD_NAME} {COMMAND} [OPTION]" echo 'Description: ' echo -e '\tThis tool allows you to mirror and keep synchronised one or more folders on a remote server with arbitraries local folders.' echo -e '\tIt has five main commands init, list, start, stop, remove' echo -e "\tNOTE: Before using it, you must have configured ssh on your remote server!" echo 'INIT: ' echo -e "\tTypical use of tool is starting a sync with the command" echo -e "\t\t${HELP_CMD_NAME} init -l ${HELP_LOCAL_DIR} -r ${HELP_REMOTE_DIR}" echo -e "\t-l option stands for the local folder you want to mirror and sync, -r option stands for the remote folder in the form known to both ssh and scp" echo 'LIST: ' echo -e "\tit shows the list of all synched local folders with respective remote folder and some other usefull information: the integer id assigned to the sync," echo -e "\tthe current status of sync, the local folder you are keeping in sync and the remote folder." echo -e "\tIn the following commands you can refer the sync to apply the command with the integer id of sync using the option -s," echo -e "\tor with the local folder using the option -l" echo 'STOP: ' echo -e "\tIf you want to stop a sync in the list of syncs, you can refer to it either by its id or by the local folder associated with the sync" echo -e "\t\t${HELP_CMD_NAME} stop -s 1 or ${HELP_CMD_NAME} stop -l ${HELP_LOCAL_DIR}" echo 'START: ' echo -e "\tIf you want to resume a stopped sync in the list of syncs, you can refer to it either by its id or by the local folder associated with the sync" echo -e "\t\t${HELP_CMD_NAME} start -s 1 or ${HELP_CMD_NAME} stop -l ${HELP_LOCAL_DIR}" echo 'REMOVE: ' echo -e "\tRemove the sync from the list of all syncs, do not delete any local folder and data" echo -e "\t\t${HELP_CMD_NAME} remove -s 1 or ${HELP_CMD_NAME} remove -l ${HELP_LOCAL_DIR}" } hastrailingslash() { case "$1" in */) echo true ;; *) echo false ;; esac } syncexists() { if [ ! -f "${SYNCFILE}" ]; then echo false return fi localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -) if [[ $(cat "${SYNCFILE}" | grep "$localpath_hash" | wc -l) -ge 1 ]]; then echo true else echo false fi } feeddeletes() { queuedeletes=$(format ${TMPQUEUEDELETES} hash=$1) snapfile=$(format ${SNAPSHOTFILE} hash=$1) grep $1 ${SYNCFILE} | while read hash pid when status localpath remotepath; do remoterelativepath=$(echo "$remotepath" | cut -d : -f 2) remotehost=$(echo "$remotepath" | cut -d : -f 1 | cut -d @ -f 2) remotefiles=$(mktemp) ## request list of remote files to remotehost ssh ${remotehost} find "'${remoterelativepath}'" > ${remotefiles} ## read my snapshot while read path; do if [[ -e "${path}" ]]; then tocheckremotepath=$(echo "${path}" | sed -e "s~${localpath}~${remoterelativepath}~g") if ! grep -Fxq "${tocheckremotepath}" ${remotefiles}; then echo -e "${PURPLE}[PURGE]${ENDCOLOR} ${path} in snapshot no longer exists in remote"; rm -rf "${path}" fi else echo -e "${PURPLE}[PURGE]${ENDCOLOR} ${path} in snapshot no longer exists in local, delete in remote"; ## if controlled file doesn't exist in local delete it on remote echo "${path}" | sed -e "s~${localpath}~${remoterelativepath}~g" | tee -a ${queuedeletes} fi done < ${snapfile} rm -rf ${remotefiles} done } ########################################## # the init action is composed by # 1. the mirroring of the remote folder. # Basycally an `rsync` that uses the remote folder as the source # and the local folder as the destination # # 2. creating a sync task (a line) in the file that contains the list of all sync # 3. starting the loop listening for the changes in local folder ########################################## initsyncpath() { echo -e "Start mirroring remote folder ${GREEN}$2${ENDCOLOR} on local folder ${GREEN}$1${ENDCOLOR}" ## write the file with remotepath and localpath for future use localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -) sync $localpath_hash $2 $1 ## create sync with status stopped -> after this start the loop echo $localpath_hash "p$MYPID" w$(date +%s) $STATUS_STOPPED $1 $2 >> ${SYNCFILE} loopsyncpath $1 } ########################################## # the start action is composed by # 1. Checking the presence of the snapshot. # Snapshot indicates the files present in origin (and in local path) when the sync was stopped # if the snapshot is present we MUST check if all file in the snapshot are still present # in origin. If not we must remove it from local folder feeding the deletes # # 2. Pushing a dummmy action into the queue to force an rsync from local folder to remote, # For eventually new files # 3. starting the loop listening for the changes in local folder ########################################## startsyncpath() { echo -e "Start synching local folder ${GREEN}$1${ENDCOLOR} with remote ${GREEN}$2${ENDCOLOR}..." localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -) # snapshotfile=${SNAPSHOTFILE}${localpath_hash} snapshotfile=$(format ${SNAPSHOTFILE} hash=${localpath_hash}) #tempqueuefile=~/.syncdir_${localpath_hash}.queue tempqueuefile=$(format ${TMPQUEUEFILE} hash=${localpath_hash}) if [ -e ${snapshotfile} ]; then echo "Founded snapshot file: ${snapshotfile} collect files to delete in origin" feeddeletes ${localpath_hash} echo "Remove snapshot file ${snapshotfile}" rm -f ${snapshotfile} fi ## ensure local computer sends eventually new files adding a fake line in the queue echo "Startsync DUMMY ACTION" >> ${tempqueuefile} echo -e "${GREEN}DONE${ENDCOLOR}" loopsyncpath $1 } readlog() { syncloopfile=$(format ${SYNCLOOPFILE} hash=$1) tail -f ${syncloopfile} } ########################################## # start the loop for synching local folder with remote folder. # You can use as first argument the integer of sync or the local path. # It spawn in background `syncloop.sh` and redirect stamdard output # and standard error on syncloop_$hash.nohup # ########################################## loopsyncpath() { echo -e "Starting loop for synching ${GREEN}$1${ENDCOLOR}" re_num='^[0-9]+$' if [[ $1 =~ $re_num && $1 -gt 0 ]]; then ## $1 is the index of line to substitute nline=$1 else ## $1 is the path of sync of line to substitute localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -) nline=$(grep -n $localpath_hash "$SYNCFILE" | cut -f1 -d ":") fi sed -n -s "$nline"p "$SYNCFILE" | while read hash pid when status localpath remotepath; do if [ $status = $STATUS_RUNNING ]; then echo 'Sync already running do nothing...' exit 0 else ## delete old rsync log file rsynclogfile=$(format ${LOGFILERSYNC} hash=$hash) rm -f ${rsynclogfile} ## starting loop on background and catch pid syncloopfile=$(format ${SYNCLOOPFILE} hash=$hash) nohup syncloop.sh $hash $localpath $remotepath 1>~/syncloop_$hash.nohup 2>&1 & mypid=$! when=$(date +%s) ## if line exists i must replace it with new pid sed -i -e "$nline {s/$STATUS_STOPPED/$STATUS_RUNNING/; s/p[0-9]\+/p$mypid/; s/w[0-9]\+/w$when/}" "$SYNCFILE" fi done } nsync() { ## find the nline of a sync starting from an hash or a number re_num='^[0-9]+$' if [[ $1 =~ $re_num && $1 -gt 0 ]]; then ## $1 is the index of line to substitute nline=$1 else ## $1 is the path of sync of line to substitute localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -) nline=$(grep -n $1 "$SYNCFILE" | cut -f1 -d ":") fi echo $nline } stoploopsyncpath() { nline=$(nsync "$1") sed -n -s "$nline"p "$SYNCFILE" | while read hash pid when status localpath remotepath; do if [ $status = $STATUS_STOPPED ]; then echo -e "${GREEN}Sync already stopped do nothing...${ENDCOLOR}" exit 0 else ## stop process and all descendants echo -e "Stop sync on local folder ${GREEN}${localpath}${ENDCOLOR} remote folder ${GREEN}${remotepath}${ENDCOLOR}..." pid=$(echo $pid | cut -c 2-) kill $(ps -o pid= --ppid $pid) when=$(date +%s) sed -i -e "$nline {s/$STATUS_RUNNING/$STATUS_STOPPED/; s/w[0-9]\+/w$when/}" "$SYNCFILE" #### SAVE THE SNAPSHOT snapshotfile=$(format ${SNAPSHOTFILE} hash=${hash}) find ${localpath} > ${snapshotfile} fi done } removeloopsyncpath() { #echo $(sed "1q;d" ~/.syncdir.sync) | sed -e "s/STOPPED/RUNNING/" nline=$(nsync "$1") sed -n -s "$nline"p "$SYNCFILE" | while read hash pid when status localpath remotepath; do if [ $status = $STATUS_RUNNING ]; then echo 'Sync is running stop it and delete from tasks..' stoploopsyncpath $nline else echo 'Sync is stopped delete it from tasks..' fi sed -i "$nline"d "$SYNCFILE" done } emptysyncfile() { if [[ $(cat ${SYNCFILE} | wc -l) -eq 0 ]]; then echo true else echo false fi } syncoutofindex() { if [[ $1 -gt $(cat ${SYNCFILE} | wc -l) ]]; then echo true else echo false fi } if [ "$1" == '-h' ]; then myhelp exit 1 fi case "$1" in init) shift while getopts ":l:r:" option; do case "${option}" in l) localpath=${OPTARG} if [ ! -d "$localpath" ]; then echo 'local path to sync not exists... do you want to create it?' test_no='^(N|n)[[:alnum:]]+' read -r ans #no " in regular expression match if [[ ${ans} =~ ${test_no} ]]; then echo '... Aborting...' exit 1 fi ## create path mkdir -p "${localpath}" fi lhts=$(hastrailingslash $localpath) if [ ${lhts} = false ]; then localpath="${localpath}"/ fi ;; r) remotepath=${OPTARG} #match an adress in this form username@from_host:/home/test re_isremote='^[[:alnum:]]+\@([[:alnum:]]+|([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})):/[[:alnum:]]+/?' if ! [[ $remotepath =~ $re_isremote ]]; then echo 'Incorrect format for destination path... aborting' exit 1 fi if [[ ${localpath} == "" ]]; then cpath=$(pwd) echo "Setting local path to ${cpath}/" localpath=${cpath}/ fi synce=$(syncexists ${localpath}) if [ ${synce} = true ]; then echo 'Folder you have specified is already synched: you can only start, stop or remove sync' cat $SYNCFILE exit 1 else rhts=$(hastrailingslash $remotepath) if [ ${rhts} = false ]; then remotepath="${remotepath}"/ fi initsyncpath $localpath $remotepath fi ;; \?) myhelp exit 1 ;; esac done ;; start) shift while getopts ":l:s:" option; do case "${option}" in l) localsync=${OPTARG} if [[ -d "$localsync" ]]; then synce=$(syncexists ${localsync}) if [ ${synce} = true ]; then echo "Sync local folder ${localsync}" grep ${localsync} $SYNCFILE | while read hash pid when status localpath remotepath; do startsyncpath ${localpath} ${remotepath} done else echo "local folder ${localsync} not in sync task, please before use init" fi else echo "ATTENTION Folder: $localsync not exists Aborting!" exit 2 fi ;; s) indexsync=${OPTARG} empty=$(emptysyncfile) if [ ${empty} = true ]; then echo "Sync ${indexsync} not exists, sync file empty, please use init first" else re_num='^[0-9]+$' if [[ ! $indexsync =~ $re_num ]]; then echo -e "-s option must be ${RED}integer${ENDCOLOR}" exit 3 fi oi=$(syncoutofindex $indexsync) if [ ${oi} = true ]; then echo "Sync ${indexsync} out of index" exit 1 else sed -n -s "$indexsync"p "$SYNCFILE" | while read hash pid when status localpath remotepath; do startsyncpath ${localpath} ${remotepath} done fi fi ;; \?) myhelp exit 1 ;; esac done ;; status) shift while getopts ":l:s:" option; do case "${option}" in l) localsync=${OPTARG} if [[ -d "$localsync" ]]; then synce=$(syncexists ${localsync}) if [ ${synce} = true ]; then echo "Read log related to local folder ${localsync}" grep ${localsync} $SYNCFILE | while read hash pid when status localpath remotepath; do readlog ${hash} done else echo "local folder ${localsync} not in sync task, you can not read the log" fi else echo "ATTENTION $localsync not exists Aborting log!" exit 2 fi ;; s) indexsync=${OPTARG} empty=$(emptysyncfile) if [ ${empty} = true ]; then echo "Sync ${indexsync} not exists, sync file empty, please use init first" else re_num='^[0-9]+$' if [[ ! $indexsync =~ $re_num ]]; then echo -e "-s option must be ${RED}integer${ENDCOLOR}" exit 3 fi oi=$(syncoutofindex $indexsync) if [ ${oi} = true ]; then echo "Sync ${indexsync} out of index" exit 1 else sed -n -s "$indexsync"p "$SYNCFILE" | while read hash pid when status localpath remotepath; do readlog ${hash} done fi fi ;; \?) myhelp exit 1 ;; esac done ;; stop) shift while getopts ":l:s:" option; do case "${option}" in l) localsync=${OPTARG} if [[ -d "$localsync" ]]; then synce=$(syncexists ${localsync}) if [ ${synce} = true ]; then stoploopsyncpath ${localsync} else echo -e "Can't stop local sync for folder ${RED}${localsync}${ENDCOLOR}. It is not in sync task, please before use init" exit 2 fi else echo -e "Local folder ${RED}${localsync}${ENDCOLOR} not exists Aborting!" exit 2 fi ;; s) indexsync=${OPTARG} empty=$(emptysyncfile) if [ ${empty} = true ]; then echo "Sync file empty, please use init first for creating a sync" exit 2 else re_num='^[0-9]+$' if [[ ! $indexsync =~ $re_num ]]; then echo -e "-s option must be ${RED}integer${ENDCOLOR}" exit 3 fi oi=$(syncoutofindex $indexsync) if [ ${oi} = true ]; then echo -e "${RED}Sync out of index${ENDCOLOR}" exit 2 else stoploopsyncpath $indexsync fi fi ;; \?) myhelp exit 1 ;; esac done ;; list) if [ ! -f $SYNCFILE ]; then echo 'Sync file empty' else if [ $(cat $SYNCFILE | wc -l) -eq 0 ]; then echo 'Sync file empty' else echo -e "$(cat $SYNCFILE | cat -n | sed -e "s/RUNNING/\\${GREEN}RUNNING\\${ENDCOLOR}/" -e "s/STOPPED/\\${RED}STOPPED\\${ENDCOLOR}/")" fi fi ;; remove) shift while getopts ":l:s:" option; do case "${option}" in l) localsync=${OPTARG} #echo "Sync this local folder: $2" synce=$(syncexists ${localsync}) if [ ${synce} = true ]; then echo "Removing Sync local folder ${localsync}" removeloopsyncpath ${localsync} else echo "Can't remove local sync for folder ${localsync}. It is not in sync task" exit 2 fi ;; s) indexsync=${OPTARG} empty=$(emptysyncfile) if [ ${empty} = true ]; then echo "Sync ${indexsync} not exists, sync file empty, please use init first" else oi=$(syncoutofindex $indexsync) if [ ${oi} = true ]; then echo "Sync ${indexsync} out of index" exit 1 else removeloopsyncpath $indexsync fi fi ;; \?) myhelp exit 1 ;; esac done ;; *) myhelp exit 1 esac