535 lines
18 KiB
Bash
Executable File
535 lines
18 KiB
Bash
Executable File
#!/bin/bash
|
|
# Copyright 2021 Luca Paris
|
|
#This file is part of masync.
|
|
|
|
#masync is free software: you can redistribute it and/or modify
|
|
#it under the terms of the GNU General Public License as published by
|
|
#the Free Software Foundation, either version 3 of the License, or
|
|
#(at your option) any later version.
|
|
|
|
#masync is distributed in the hope that it will be useful,
|
|
#but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
#GNU General Public License for more details.
|
|
|
|
#You should have received a copy of the GNU General Public License
|
|
#along with masync. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
source ~/bin/.synccmd.sh
|
|
source ~/bin/.filetemplates.sh
|
|
source ~/bin/.colordef.sh
|
|
MYPID=$$
|
|
STATUS_RUNNING=RUNNING
|
|
STATUS_STOPPED=STOPPED
|
|
|
|
HELP_CMD_NAME='masync'
|
|
HELP_LOCAL_DIR="/home/$USER/localsync/"
|
|
HELP_REMOTE_DIR="remoteuser@server:/home/remoteuser/sync/"
|
|
|
|
myhelp() {
|
|
echo "Usage: ${HELP_CMD_NAME} {COMMAND} [OPTION]"
|
|
echo 'Description: '
|
|
echo -e '\tThis tool allows you to mirror and keep synchronised one or more folders on a remote server with arbitraries local folders.'
|
|
echo -e '\tIt has five main commands init, list, start, stop, remove'
|
|
echo -e "\tNOTE: Before using it, you must have configured ssh on your remote server!"
|
|
echo 'INIT: '
|
|
echo -e "\tTypical use of tool is starting a sync with the command"
|
|
echo -e "\t\t${HELP_CMD_NAME} init -l ${HELP_LOCAL_DIR} -r ${HELP_REMOTE_DIR}"
|
|
echo -e "\t-l option stands for the local folder you want to mirror and sync, -r option stands for the remote folder in the form known to both ssh and scp"
|
|
echo 'LIST: '
|
|
echo -e "\tit shows the list of all synched local folders with respective remote folder and some other usefull information: the integer id assigned to the sync,"
|
|
echo -e "\tthe current status of sync, the local folder you are keeping in sync and the remote folder."
|
|
echo -e "\tIn the following commands you can refer the sync to apply the command with the integer id of sync using the option -s,"
|
|
echo -e "\tor with the local folder using the option -l"
|
|
echo 'STOP: '
|
|
echo -e "\tIf you want to stop a sync in the list of syncs, you can refer to it either by its id or by the local folder associated with the sync"
|
|
echo -e "\t\t${HELP_CMD_NAME} stop -s 1 or ${HELP_CMD_NAME} stop -l ${HELP_LOCAL_DIR}"
|
|
echo 'START: '
|
|
echo -e "\tIf you want to resume a stopped sync in the list of syncs, you can refer to it either by its id or by the local folder associated with the sync"
|
|
echo -e "\t\t${HELP_CMD_NAME} start -s 1 or ${HELP_CMD_NAME} stop -l ${HELP_LOCAL_DIR}"
|
|
echo 'REMOVE: '
|
|
echo -e "\tRemove the sync from the list of all syncs, do not delete any local folder and data"
|
|
echo -e "\t\t${HELP_CMD_NAME} remove -s 1 or ${HELP_CMD_NAME} remove -l ${HELP_LOCAL_DIR}"
|
|
}
|
|
|
|
hastrailingslash() {
|
|
case "$1" in
|
|
*/)
|
|
echo true
|
|
;;
|
|
*)
|
|
echo false
|
|
;;
|
|
esac
|
|
}
|
|
|
|
syncexists() {
|
|
if [ ! -f "${SYNCFILE}" ]; then
|
|
echo false
|
|
return
|
|
fi
|
|
localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -)
|
|
if [[ $(cat "${SYNCFILE}" | grep "$localpath_hash" | wc -l) -ge 1 ]]; then
|
|
echo true
|
|
else
|
|
echo false
|
|
fi
|
|
}
|
|
|
|
feeddeletes() {
|
|
queuedeletes=$(format ${TMPQUEUEDELETES} hash=$1)
|
|
snapfile=$(format ${SNAPSHOTFILE} hash=$1)
|
|
grep $1 ${SYNCFILE} | while read hash pid when status localpath remotepath; do
|
|
remoterelativepath=$(echo "$remotepath" | cut -d : -f 2)
|
|
remotehost=$(echo "$remotepath" | cut -d : -f 1 | cut -d @ -f 2)
|
|
remotefiles=$(mktemp)
|
|
## request list of remote files to remotehost
|
|
ssh ${remotehost} find "'${remoterelativepath}'" > ${remotefiles}
|
|
## read my snapshot
|
|
while read path; do
|
|
if [[ -e "${path}" ]]; then
|
|
tocheckremotepath=$(echo "${path}" | sed -e "s~${localpath}~${remoterelativepath}~g")
|
|
if ! grep -Fxq "${tocheckremotepath}" ${remotefiles}; then
|
|
# echo -e "${PURPLE}[PURGE]${ENDCOLOR} ${path} in snapshot no longer exists in remote"
|
|
echo -e -n "Do you want to remove ${RED}${path}${ENDCOLOR} from local sync? [y to remove, return to skip it, N no to all] "
|
|
read -u 1 ui
|
|
if [[ -z "${ui}" ]]; then
|
|
if [ ${ui} = 'y' ]; then
|
|
rm -rf "${path}"
|
|
fi
|
|
if [ ${ui} = 'N' ]; then
|
|
break
|
|
fi
|
|
fi
|
|
fi
|
|
else
|
|
# echo -e "${PURPLE}[PURGE]${ENDCOLOR} ${path} in snapshot no longer exists in local, delete in remote"
|
|
echo -e -n "Do you want to remove ${RED}${path}${ENDCOLOR} from remote origin? [y to remove, return to skip it, N no to all] "
|
|
read -u 1 ui
|
|
if [[ -z "${ui}" ]]; then
|
|
if [ ${ui} = 'y' ]; then
|
|
## if controlled file doesn't exist in local delete it on remote
|
|
echo "${path}" | sed -e "s~${localpath}~${remoterelativepath}~g" | tee -a ${queuedeletes}
|
|
fi
|
|
if [ ${ui} = 'N' ]; then
|
|
break
|
|
fi
|
|
fi
|
|
fi
|
|
done < ${snapfile}
|
|
rm -rf ${remotefiles}
|
|
done
|
|
}
|
|
|
|
##########################################
|
|
# the init action is composed by
|
|
# 1. the mirroring of the remote folder.
|
|
# Basycally an `rsync` that uses the remote folder as the source
|
|
# and the local folder as the destination
|
|
#
|
|
# 2. creating a sync task (a line) in the file that contains the list of all sync
|
|
# 3. starting the loop listening for the changes in local folder
|
|
##########################################
|
|
|
|
initsyncpath() {
|
|
echo -e "Start mirroring remote folder ${GREEN}$2${ENDCOLOR} on local folder ${GREEN}$1${ENDCOLOR}"
|
|
## write the file with remotepath and localpath for future use
|
|
localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -)
|
|
sync $localpath_hash $2 $1
|
|
## create sync with status stopped -> after this start the loop
|
|
echo $localpath_hash "p$MYPID" w$(date +%s) $STATUS_STOPPED $1 $2 >> ${SYNCFILE}
|
|
loopsyncpath $1
|
|
|
|
}
|
|
|
|
##########################################
|
|
# the start action is composed by
|
|
# 1. Checking the presence of the snapshot.
|
|
# Snapshot indicates the files present in origin (and in local path) when the sync was stopped
|
|
# if the snapshot is present we MUST check if all file in the snapshot are still present
|
|
# in origin. If not we must remove it from local folder feeding the deletes
|
|
#
|
|
# 2. Pushing a dummmy action into the queue to force an rsync from local folder to remote,
|
|
# For eventually new files
|
|
# 3. starting the loop listening for the changes in local folder
|
|
##########################################
|
|
|
|
startsyncpath() {
|
|
echo -e "Start synching local folder ${GREEN}$1${ENDCOLOR} with remote ${GREEN}$2${ENDCOLOR}..."
|
|
localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -)
|
|
# snapshotfile=${SNAPSHOTFILE}${localpath_hash}
|
|
snapshotfile=$(format ${SNAPSHOTFILE} hash=${localpath_hash})
|
|
#tempqueuefile=~/.syncdir_${localpath_hash}.queue
|
|
tempqueuefile=$(format ${TMPQUEUEFILE} hash=${localpath_hash})
|
|
syncloopfile=$(format ${SYNCLOOPFILE} hash=${localpath_hash})
|
|
|
|
if [ -e ${snapshotfile} ]; then
|
|
echo "Founded snapshot file: ${snapshotfile} checking files to remove..."
|
|
feeddeletes ${localpath_hash}
|
|
echo "Remove snapshot file ${snapshotfile}"
|
|
rm -f ${snapshotfile}
|
|
fi
|
|
## ensure local computer sends eventually new files adding a fake line in the queue
|
|
echo -e "${PURPLE}[DUMMY PUSH]${ENDCOLOR} to remote" >> ${syncloopfile}
|
|
echo "Startsync DUMMY ACTION" >> ${tempqueuefile}
|
|
echo -e "${GREEN}DONE${ENDCOLOR}"
|
|
loopsyncpath $1
|
|
}
|
|
|
|
readlog() {
|
|
syncloopfile=$(format ${SYNCLOOPFILE} hash=$1)
|
|
tail -f ${syncloopfile}
|
|
}
|
|
|
|
##########################################
|
|
# start the loop for synching local folder with remote folder.
|
|
# You can use as first argument the integer of sync or the local path.
|
|
# It spawn in background `syncloop.sh` and redirect stamdard output
|
|
# and standard error on syncloop_$hash.nohup
|
|
#
|
|
##########################################
|
|
|
|
loopsyncpath() {
|
|
echo -e "Starting loop for synching ${GREEN}$1${ENDCOLOR}"
|
|
re_num='^[0-9]+$'
|
|
if [[ $1 =~ $re_num && $1 -gt 0 ]]; then
|
|
## $1 is the index of line to substitute
|
|
nline=$1
|
|
else
|
|
## $1 is the path of sync of line to substitute
|
|
localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -)
|
|
nline=$(grep -n $localpath_hash "$SYNCFILE" | cut -f1 -d ":")
|
|
fi
|
|
sed -n -s "$nline"p "$SYNCFILE" | while read hash pid when status localpath remotepath; do
|
|
if [ $status = $STATUS_RUNNING ]; then
|
|
echo 'Sync already running do nothing...'
|
|
exit 0
|
|
else
|
|
## delete old rsync log file
|
|
rsynclogfile=$(format ${LOGFILERSYNC} hash=$hash)
|
|
rm -f ${rsynclogfile}
|
|
## starting loop on background and catch pid
|
|
syncloopfile=$(format ${SYNCLOOPFILE} hash=$hash)
|
|
nohup syncloop.sh $hash $localpath $remotepath 1>$syncloopfile 2>&1 &
|
|
mypid=$!
|
|
when=$(date +%s)
|
|
## if line exists i must replace it with new pid
|
|
sed -i -e "$nline {s/$STATUS_STOPPED/$STATUS_RUNNING/; s/p[0-9]\+/p$mypid/; s/w[0-9]\+/w$when/}" "$SYNCFILE"
|
|
fi
|
|
done
|
|
}
|
|
|
|
nsync() {
|
|
## find the nline of a sync starting from an hash or a number
|
|
re_num='^[0-9]+$'
|
|
if [[ $1 =~ $re_num && $1 -gt 0 ]]; then
|
|
## $1 is the index of line to substitute
|
|
nline=$1
|
|
else
|
|
## $1 is the path of sync of line to substitute
|
|
localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -)
|
|
nline=$(grep -n $1 "$SYNCFILE" | cut -f1 -d ":")
|
|
fi
|
|
echo $nline
|
|
}
|
|
|
|
stoploopsyncpath() {
|
|
nline=$(nsync "$1")
|
|
sed -n -s "$nline"p "$SYNCFILE" | while read hash pid when status localpath remotepath; do
|
|
if [ $status = $STATUS_STOPPED ]; then
|
|
echo -e "${GREEN}Sync already stopped do nothing...${ENDCOLOR}"
|
|
exit 0
|
|
else
|
|
## stop process and all descendants
|
|
echo -e "Stop sync on local folder ${GREEN}${localpath}${ENDCOLOR} remote folder ${GREEN}${remotepath}${ENDCOLOR}..."
|
|
pid=$(echo $pid | cut -c 2-)
|
|
kill $(ps -o pid= --ppid $pid)
|
|
when=$(date +%s)
|
|
sed -i -e "$nline {s/$STATUS_RUNNING/$STATUS_STOPPED/; s/w[0-9]\+/w$when/}" "$SYNCFILE"
|
|
#### SAVE THE SNAPSHOT
|
|
snapshotfile=$(format ${SNAPSHOTFILE} hash=${hash})
|
|
find ${localpath} > ${snapshotfile}
|
|
fi
|
|
done
|
|
}
|
|
|
|
removeloopsyncpath() {
|
|
#echo $(sed "1q;d" ~/.syncdir.sync) | sed -e "s/STOPPED/RUNNING/"
|
|
nline=$(nsync "$1")
|
|
sed -n -s "$nline"p "$SYNCFILE" | while read hash pid when status localpath remotepath; do
|
|
if [ $status = $STATUS_RUNNING ]; then
|
|
echo 'Sync is running stop it and delete from tasks..'
|
|
stoploopsyncpath $nline
|
|
else
|
|
echo 'Sync is stopped delete it from tasks..'
|
|
fi
|
|
sed -i "$nline"d "$SYNCFILE"
|
|
done
|
|
}
|
|
|
|
emptysyncfile() {
|
|
if [[ $(cat ${SYNCFILE} | wc -l) -eq 0 ]]; then
|
|
echo true
|
|
else
|
|
echo false
|
|
fi
|
|
}
|
|
|
|
syncoutofindex() {
|
|
if [[ $1 -gt $(cat ${SYNCFILE} | wc -l) ]]; then
|
|
echo true
|
|
else
|
|
echo false
|
|
fi
|
|
}
|
|
|
|
if [ "$1" == '-h' ]; then
|
|
myhelp
|
|
exit 1
|
|
fi
|
|
|
|
case "$1" in
|
|
init)
|
|
shift
|
|
while getopts ":l:r:" option; do
|
|
case "${option}" in
|
|
l) localpath=${OPTARG}
|
|
if [ ! -d "$localpath" ]; then
|
|
echo 'local path to sync not exists... do you want to create it?'
|
|
test_no='^(N|n)[[:alnum:]]+'
|
|
read -r ans
|
|
#no " in regular expression match
|
|
if [[ ${ans} =~ ${test_no} ]]; then
|
|
echo '... Aborting...'
|
|
exit 1
|
|
fi
|
|
## create path
|
|
mkdir -p "${localpath}"
|
|
fi
|
|
lhts=$(hastrailingslash $localpath)
|
|
if [ ${lhts} = false ]; then
|
|
localpath="${localpath}"/
|
|
fi
|
|
;;
|
|
r) remotepath=${OPTARG}
|
|
#match an adress in this form username@from_host:/home/test
|
|
re_isremote='^[[:alnum:]]+\@([[:alnum:]]+|([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})):/[[:alnum:]]+/?'
|
|
if ! [[ $remotepath =~ $re_isremote ]]; then
|
|
echo 'Incorrect format for destination path... aborting'
|
|
exit 1
|
|
fi
|
|
if [[ ${localpath} == "" ]]; then
|
|
cpath=$(pwd)
|
|
echo "Setting local path to ${cpath}/"
|
|
localpath=${cpath}/
|
|
fi
|
|
synce=$(syncexists ${localpath})
|
|
if [ ${synce} = true ]; then
|
|
echo 'Folder you have specified is already synched: you can only start, stop or remove sync'
|
|
cat $SYNCFILE
|
|
exit 1
|
|
else
|
|
rhts=$(hastrailingslash $remotepath)
|
|
if [ ${rhts} = false ]; then
|
|
remotepath="${remotepath}"/
|
|
fi
|
|
initsyncpath $localpath $remotepath
|
|
fi
|
|
|
|
;;
|
|
\?) myhelp
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
;;
|
|
start)
|
|
shift
|
|
while getopts ":l:s:" option; do
|
|
case "${option}" in
|
|
l) localsync=${OPTARG}
|
|
if [[ -d "$localsync" ]]; then
|
|
synce=$(syncexists ${localsync})
|
|
if [ ${synce} = true ]; then
|
|
echo "Sync local folder ${localsync}"
|
|
grep ${localsync} $SYNCFILE | while read hash pid when status localpath remotepath; do
|
|
startsyncpath ${localpath} ${remotepath}
|
|
done
|
|
else
|
|
echo "local folder ${localsync} not in sync task, please before use init"
|
|
fi
|
|
else
|
|
echo "ATTENTION Folder: $localsync not exists Aborting!"
|
|
exit 2
|
|
fi
|
|
;;
|
|
s) indexsync=${OPTARG}
|
|
empty=$(emptysyncfile)
|
|
if [ ${empty} = true ]; then
|
|
echo "Sync ${indexsync} not exists, sync file empty, please use init first"
|
|
else
|
|
re_num='^[0-9]+$'
|
|
if [[ ! $indexsync =~ $re_num ]]; then
|
|
echo -e "-s option must be ${RED}integer${ENDCOLOR}"
|
|
exit 3
|
|
fi
|
|
oi=$(syncoutofindex $indexsync)
|
|
if [ ${oi} = true ]; then
|
|
echo "Sync ${indexsync} out of index"
|
|
exit 1
|
|
else
|
|
sed -n -s "$indexsync"p "$SYNCFILE" | while read hash pid when status localpath remotepath; do
|
|
startsyncpath ${localpath} ${remotepath}
|
|
done
|
|
fi
|
|
fi
|
|
;;
|
|
\?) myhelp
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
;;
|
|
status)
|
|
shift
|
|
while getopts ":l:s:" option; do
|
|
case "${option}" in
|
|
l) localsync=${OPTARG}
|
|
if [[ -d "$localsync" ]]; then
|
|
synce=$(syncexists ${localsync})
|
|
if [ ${synce} = true ]; then
|
|
echo "Read log related to local folder ${localsync}"
|
|
grep ${localsync} $SYNCFILE | while read hash pid when status localpath remotepath; do
|
|
readlog ${hash}
|
|
done
|
|
else
|
|
echo "local folder ${localsync} not in sync task, you can not read the log"
|
|
fi
|
|
else
|
|
echo "ATTENTION $localsync not exists Aborting log!"
|
|
exit 2
|
|
fi
|
|
;;
|
|
s) indexsync=${OPTARG}
|
|
empty=$(emptysyncfile)
|
|
if [ ${empty} = true ]; then
|
|
echo "Sync ${indexsync} not exists, sync file empty, please use init first"
|
|
else
|
|
re_num='^[0-9]+$'
|
|
if [[ ! $indexsync =~ $re_num ]]; then
|
|
echo -e "-s option must be ${RED}integer${ENDCOLOR}"
|
|
exit 3
|
|
fi
|
|
oi=$(syncoutofindex $indexsync)
|
|
if [ ${oi} = true ]; then
|
|
echo "Sync ${indexsync} out of index"
|
|
exit 1
|
|
else
|
|
sed -n -s "$indexsync"p "$SYNCFILE" | while read hash pid when status localpath remotepath; do
|
|
readlog ${hash}
|
|
done
|
|
fi
|
|
fi
|
|
;;
|
|
\?) myhelp
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
;;
|
|
stop)
|
|
shift
|
|
while getopts ":l:s:" option; do
|
|
case "${option}" in
|
|
l) localsync=${OPTARG}
|
|
if [[ -d "$localsync" ]]; then
|
|
synce=$(syncexists ${localsync})
|
|
if [ ${synce} = true ]; then
|
|
stoploopsyncpath ${localsync}
|
|
else
|
|
echo -e "Can't stop local sync for folder ${RED}${localsync}${ENDCOLOR}. It is not in sync task, please before use init"
|
|
exit 2
|
|
fi
|
|
else
|
|
echo -e "Local folder ${RED}${localsync}${ENDCOLOR} not exists Aborting!"
|
|
exit 2
|
|
fi
|
|
;;
|
|
s) indexsync=${OPTARG}
|
|
empty=$(emptysyncfile)
|
|
if [ ${empty} = true ]; then
|
|
echo "Sync file empty, please use init first for creating a sync"
|
|
exit 2
|
|
else
|
|
re_num='^[0-9]+$'
|
|
if [[ ! $indexsync =~ $re_num ]]; then
|
|
echo -e "-s option must be ${RED}integer${ENDCOLOR}"
|
|
exit 3
|
|
fi
|
|
oi=$(syncoutofindex $indexsync)
|
|
if [ ${oi} = true ]; then
|
|
echo -e "${RED}Sync out of index${ENDCOLOR}"
|
|
exit 2
|
|
else
|
|
stoploopsyncpath $indexsync
|
|
fi
|
|
fi
|
|
;;
|
|
\?) myhelp
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
;;
|
|
list)
|
|
if [ ! -f $SYNCFILE ]; then
|
|
echo 'Sync file empty'
|
|
else
|
|
if [ $(cat $SYNCFILE | wc -l) -eq 0 ]; then
|
|
echo 'Sync file empty'
|
|
else
|
|
echo -e "$(cat $SYNCFILE | cat -n | sed -e "s/RUNNING/\\${GREEN}RUNNING\\${ENDCOLOR}/" -e "s/STOPPED/\\${RED}STOPPED\\${ENDCOLOR}/")"
|
|
fi
|
|
fi
|
|
;;
|
|
remove)
|
|
shift
|
|
while getopts ":l:s:" option; do
|
|
case "${option}" in
|
|
l) localsync=${OPTARG}
|
|
#echo "Sync this local folder: $2"
|
|
synce=$(syncexists ${localsync})
|
|
if [ ${synce} = true ]; then
|
|
echo "Removing Sync local folder ${localsync}"
|
|
removeloopsyncpath ${localsync}
|
|
else
|
|
echo "Can't remove local sync for folder ${localsync}. It is not in sync task"
|
|
exit 2
|
|
fi
|
|
;;
|
|
s) indexsync=${OPTARG}
|
|
empty=$(emptysyncfile)
|
|
if [ ${empty} = true ]; then
|
|
echo "Sync ${indexsync} not exists, sync file empty, please use init first"
|
|
else
|
|
oi=$(syncoutofindex $indexsync)
|
|
if [ ${oi} = true ]; then
|
|
echo "Sync ${indexsync} out of index"
|
|
exit 1
|
|
else
|
|
removeloopsyncpath $indexsync
|
|
fi
|
|
fi
|
|
;;
|
|
\?) myhelp
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
;;
|
|
*)
|
|
myhelp
|
|
exit 1
|
|
esac
|