masy/masync

484 lines
16 KiB
Plaintext
Raw Normal View History

2021-06-09 17:04:43 +02:00
#!/bin/bash
2021-06-12 21:48:13 +02:00
# Copyright 2021 Luca Paris
#This file is part of masync.
#masync is free software: you can redistribute it and/or modify
#it under the terms of the GNU General Public License as published by
#the Free Software Foundation, either version 3 of the License, or
#(at your option) any later version.
#masync is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#GNU General Public License for more details.
#You should have received a copy of the GNU General Public License
#along with masync. If not, see <http://www.gnu.org/licenses/>.
2021-06-09 17:04:43 +02:00
source ~/bin/.synccmd.sh
source ~/bin/.colordef.sh
2021-06-09 17:04:43 +02:00
SYNCFILE=~/.syncdir.sync
2021-07-26 16:39:59 +02:00
SNAPSHOTFILE=~/.snapshot_
2021-06-09 17:04:43 +02:00
MYPID=$$
STATUS_RUNNING=RUNNING
STATUS_STOPPED=STOPPED
2021-06-12 21:48:13 +02:00
HELP_CMD_NAME='masync'
2021-06-10 09:33:11 +02:00
HELP_LOCAL_DIR="/home/$USER/localsync/"
HELP_REMOTE_DIR="remoteuser@server:/home/remoteuser/sync/"
2021-06-09 17:04:43 +02:00
myhelp() {
2021-06-10 14:19:57 +02:00
echo "Usage: ${HELP_CMD_NAME} {COMMAND} [OPTION]"
2021-06-09 17:04:43 +02:00
echo 'Description: '
2021-06-12 21:48:13 +02:00
echo -e '\tThis tool allows you to mirror and keep synchronised one or more folders on a remote server with arbitraries local folders.'
2021-06-14 21:31:43 +02:00
echo -e '\tIt has five main commands init, list, start, stop, remove'
2021-06-12 21:48:13 +02:00
echo -e "\tNOTE: Before using it, you must have configured ssh on your remote server!"
2021-06-14 17:09:11 +02:00
echo 'INIT: '
echo -e "\tTypical use of tool is starting a sync with the command"
2021-06-09 17:04:43 +02:00
echo -e "\t\t${HELP_CMD_NAME} init -l ${HELP_LOCAL_DIR} -r ${HELP_REMOTE_DIR}"
2021-06-14 17:09:11 +02:00
echo -e "\t-l option stands for the local folder you want to mirror and sync, -r option stands for the remote folder in the form known to both ssh and scp"
echo 'LIST: '
echo -e "\tit shows the list of all synched local folders with respective remote folder and some other usefull information: the integer id assigned to the sync,"
echo -e "\tthe current status of sync, the local folder you are keeping in sync and the remote folder."
2021-06-16 22:10:16 +02:00
echo -e "\tIn the following commands you can refer the sync to apply the command with the integer id of sync using the option -s,"
2021-06-14 17:09:11 +02:00
echo -e "\tor with the local folder using the option -l"
echo 'STOP: '
echo -e "\tIf you want to stop a sync in the list of syncs, you can refer to it either by its id or by the local folder associated with the sync"
echo -e "\t\t${HELP_CMD_NAME} stop -s 1 or ${HELP_CMD_NAME} stop -l ${HELP_LOCAL_DIR}"
echo 'START: '
echo -e "\tIf you want to resume a stopped sync in the list of syncs, you can refer to it either by its id or by the local folder associated with the sync"
echo -e "\t\t${HELP_CMD_NAME} start -s 1 or ${HELP_CMD_NAME} stop -l ${HELP_LOCAL_DIR}"
echo 'REMOVE: '
echo -e "\tRemove the sync from the list of all syncs, do not delete any local folder and data"
2021-06-14 21:31:43 +02:00
echo -e "\t\t${HELP_CMD_NAME} remove -s 1 or ${HELP_CMD_NAME} remove -l ${HELP_LOCAL_DIR}"
2021-06-09 17:04:43 +02:00
}
2021-06-10 09:33:11 +02:00
hastrailingslash() {
case "$1" in
*/)
echo true
;;
*)
echo false
;;
esac
}
2021-06-09 17:04:43 +02:00
syncexists() {
2021-06-22 10:49:48 +02:00
if [ ! -f "${SYNCFILE}" ]; then
echo false
2021-06-22 10:49:48 +02:00
return
fi
2021-06-09 17:04:43 +02:00
localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -)
if [[ $(cat "${SYNCFILE}" | grep "$localpath_hash" | wc -l) -ge 1 ]]; then
echo true
else
echo false
fi
}
2021-07-26 16:39:59 +02:00
feeddeletes() {
queuedeletes=~/.syncdir_$1.deletes
snapfile=${SNAPSHOTFILE}$1
grep $1 ${SYNCFILE} | while read hash pid when status localpath remotepath; do
remoterelativepath=$(echo "$remotepath" | cut -d : -f 2)
2021-10-06 08:00:27 +02:00
remotehost=$(echo "$remotepath" | cut -d : -f 1 | cut -d @ -f 2)
remotefiles=$(mktemp)
## request list of remote files to remotehost
ssh ${remotehost} find "'${remoterelativepath}'" > ${remotefiles}
## read my snapshot
2021-07-26 16:39:59 +02:00
while read path; do
2021-10-06 08:00:27 +02:00
if [[ -e "${path}" ]]; then
tocheckremotepath=$(echo "${path}" | sed -e "s~${localpath}~${remoterelativepath}~g")
if ! grep -Fxq "${tocheckremotepath}" ${remotefiles}; then
2022-03-18 17:01:23 +01:00
echo -e "${PURPLE}[PURGE]${ENDCOLOR} ${path} in snapshot no longer exists in remote";
2021-10-06 08:00:27 +02:00
rm -rf "${path}"
fi
else
2022-03-18 17:01:23 +01:00
echo -e "${PURPLE}[PURGE]${ENDCOLOR} ${path} in snapshot no longer exists in local, delete in remote";
2021-10-06 08:00:27 +02:00
## if controlled file doesn't exist in local delete it on remote
echo "${path}" | sed -e "s~${localpath}~${remoterelativepath}~g" | tee -a ${queuedeletes}
2021-07-26 16:39:59 +02:00
fi
done < ${snapfile}
2021-10-06 08:00:27 +02:00
rm -rf ${remotefiles}
2021-07-26 16:39:59 +02:00
done
}
2021-06-09 17:04:43 +02:00
initsyncpath() {
echo 'Starting initial sync...'
## write the file with remotepath and localpath for future use
localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -)
sync $localpath_hash $2 $1
## create sync with status stopped -> after this start the loop
echo $localpath_hash "p$MYPID" w$(date +%s) $STATUS_STOPPED $1 $2 >> ${SYNCFILE}
loopsyncpath $1
}
startsyncpath() {
2021-06-12 22:33:52 +02:00
echo -e "Starting mirror local folder ${GREEN}$1${ENDCOLOR} with remote ${GREEN}$2${ENDCOLOR}..."
2021-06-09 17:04:43 +02:00
localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -)
2021-07-26 16:39:59 +02:00
snapshotfile=${SNAPSHOTFILE}${localpath_hash}
2021-07-27 09:47:01 +02:00
tempqueuefile=~/.syncdir_${localpath_hash}.queue
2021-07-26 16:39:59 +02:00
if [ -e ${snapshotfile} ]; then
echo "Founded snapshot file: ${snapshotfile} collect files to delete in origin"
feeddeletes ${localpath_hash}
echo "Remove snapshot file ${snapshotfile}"
rm -f ${snapshotfile}
fi
2021-07-27 09:47:01 +02:00
## ensure local computer sends eventually new files adding a fake line in the queue
2022-04-18 08:58:30 +02:00
echo "Startsync DUMMY ACTION" >> ${tempqueuefile}
2022-03-18 17:01:23 +01:00
echo -e "Mirrorring ${GREEN}DONE${ENDCOLOR}"
2021-06-09 17:04:43 +02:00
loopsyncpath $1
}
2022-02-22 17:04:51 +01:00
readlog() {
tail -f ~/syncloop_$1.nohup
}
2021-06-09 17:04:43 +02:00
loopsyncpath() {
2021-06-12 22:33:52 +02:00
echo -e "Starting loop for synching ${GREEN}$1${ENDCOLOR}"
2021-06-09 17:04:43 +02:00
re_num='^[0-9]+$'
if [[ $1 =~ $re_num && $1 -gt 0 ]]; then
## $1 is the index of line to substitute
nline=$1
else
## $1 is the path of sync of line to substitute
localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -)
nline=$(grep -n $localpath_hash "$SYNCFILE" | cut -f1 -d ":")
fi
sed -n -s "$nline"p "$SYNCFILE" | while read hash pid when status localpath remotepath; do
if [ $status = $STATUS_RUNNING ]; then
echo 'Sync already running do nothing...'
exit 0
else
2021-08-23 15:08:14 +02:00
## delete old log file
rm -f ~/syncdir_$hash.log
2021-06-09 17:04:43 +02:00
## starting loop on background and catch pid
nohup syncloop.sh $hash $localpath $remotepath 1>~/syncloop_$hash.nohup 2>&1 &
mypid=$!
when=$(date +%s)
2021-06-12 22:33:52 +02:00
## if line exists i must replace it with new pid
2021-06-09 17:04:43 +02:00
sed -i -e "$nline {s/$STATUS_STOPPED/$STATUS_RUNNING/; s/p[0-9]\+/p$mypid/; s/w[0-9]\+/w$when/}" "$SYNCFILE"
fi
done
}
nsync() {
## find the nline of a sync starting from an hash or a number
re_num='^[0-9]+$'
if [[ $1 =~ $re_num && $1 -gt 0 ]]; then
## $1 is the index of line to substitute
nline=$1
else
## $1 is the path of sync of line to substitute
localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -)
nline=$(grep -n $1 "$SYNCFILE" | cut -f1 -d ":")
fi
echo $nline
}
stoploopsyncpath() {
nline=$(nsync "$1")
sed -n -s "$nline"p "$SYNCFILE" | while read hash pid when status localpath remotepath; do
if [ $status = $STATUS_STOPPED ]; then
2021-06-12 22:33:52 +02:00
echo -e "${GREEN}Sync already stopped do nothing...${ENDCOLOR}"
2021-06-09 17:04:43 +02:00
exit 0
else
## stop process and all descendants
2021-06-12 22:33:52 +02:00
echo -e "Stop sync on local folder ${GREEN}${localpath}${ENDCOLOR} remote folder ${GREEN}${remotepath}${ENDCOLOR}..."
2021-06-09 17:04:43 +02:00
pid=$(echo $pid | cut -c 2-)
kill $(ps -o pid= --ppid $pid)
when=$(date +%s)
sed -i -e "$nline {s/$STATUS_RUNNING/$STATUS_STOPPED/; s/w[0-9]\+/w$when/}" "$SYNCFILE"
2021-07-26 16:39:59 +02:00
#### SAVE THE SNAPSHOT
find ${localpath} > ${SNAPSHOTFILE}${hash}
2021-06-09 17:04:43 +02:00
fi
done
}
removeloopsyncpath() {
echo "removeloopsyncpath $1"
#echo $(sed "1q;d" ~/.syncdir.sync) | sed -e "s/STOPPED/RUNNING/"
nline=$(nsync "$1")
sed -n -s "$nline"p "$SYNCFILE" | while read hash pid when status localpath remotepath; do
if [ $status = $STATUS_RUNNING ]; then
echo 'Sync is running stop it and delete from tasks..'
stoploopsyncpath $nline
else
echo 'Sync is stopped delete it from tasks..'
fi
sed -i "$nline"d "$SYNCFILE"
done
}
emptysyncfile() {
if [[ $(cat ${SYNCFILE} | wc -l) -eq 0 ]]; then
echo true
else
echo false
fi
}
syncoutofindex() {
if [[ $1 -gt $(cat ${SYNCFILE} | wc -l) ]]; then
echo true
else
echo false
fi
}
if [ "$1" == '-h' ]; then
myhelp
exit 1
fi
case "$1" in
init)
shift
while getopts ":l:r:" option; do
case "${option}" in
l) echo "l option: ${option} value ${OPTARG}"
localpath=${OPTARG}
if [ ! -d "$localpath" ]; then
echo 'local path to sync not exists... do you want to create it?'
test_no='^(N|n)[[:alnum:]]+'
read -r ans
#no " in regular expression match
if [[ ${ans} =~ ${test_no} ]]; then
echo '... Aborting...'
exit 1
fi
## create path
mkdir -p "${localpath}"
fi
2021-06-10 09:33:11 +02:00
lhts=$(hastrailingslash $localpath)
if [ ${lhts} = false ]; then
localpath="${localpath}"/
fi
2021-06-09 17:04:43 +02:00
;;
r) echo "r option: ${option} value ${OPTARG}"
#match an adress in this form username@from_host:/home/test
remotepath=${OPTARG}
re_isremote='^[[:alnum:]]+\@([[:alnum:]]+|([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})):/[[:alnum:]]+/?'
if ! [[ $remotepath =~ $re_isremote ]]; then
echo 'Incorrect format for destination path... aborting'
exit 1
fi
if [[ ${localpath} == "" ]]; then
cpath=$(pwd)
echo "Setting local path to ${cpath}/"
localpath=${cpath}/
fi
synce=$(syncexists ${localpath})
if [ ${synce} = true ]; then
echo 'Folder you have specified is already synched: you can only start, stop or remove sync'
cat $SYNCFILE
exit 1
else
2021-06-10 09:33:11 +02:00
rhts=$(hastrailingslash $remotepath)
if [ ${rhts} = false ]; then
remotepath="${remotepath}"/
fi
2021-06-09 17:04:43 +02:00
initsyncpath $localpath $remotepath
fi
;;
\?) myhelp
exit 1
;;
esac
done
;;
start)
shift
while getopts ":l:s:" option; do
case "${option}" in
l) localsync=${OPTARG}
if [[ -d "$localsync" ]]; then
synce=$(syncexists ${localsync})
if [ ${synce} = true ]; then
echo "Sync local folder ${localsync}"
grep ${localsync} $SYNCFILE | while read hash pid when status localpath remotepath; do
startsyncpath ${localpath} ${remotepath}
done
else
echo "local folder ${localsync} not in sync task, please before use init"
fi
else
echo "ATTENTION Folder: $localsync not exists Aborting!"
exit 2
fi
;;
s) indexsync=${OPTARG}
empty=$(emptysyncfile)
if [ ${empty} = true ]; then
echo "Sync ${indexsync} not exists, sync file empty, please use init first"
else
2021-06-12 22:33:52 +02:00
re_num='^[0-9]+$'
if [[ ! $indexsync =~ $re_num ]]; then
echo -e "-s option must be ${RED}integer${ENDCOLOR}"
exit 3
fi
2021-06-09 17:04:43 +02:00
oi=$(syncoutofindex $indexsync)
if [ ${oi} = true ]; then
echo "Sync ${indexsync} out of index"
exit 1
else
sed -n -s "$indexsync"p "$SYNCFILE" | while read hash pid when status localpath remotepath; do
startsyncpath ${localpath} ${remotepath}
done
fi
fi
;;
\?) myhelp
exit 1
;;
esac
done
;;
2022-02-22 17:04:51 +01:00
status)
shift
while getopts ":l:s:" option; do
case "${option}" in
l) localsync=${OPTARG}
if [[ -d "$localsync" ]]; then
synce=$(syncexists ${localsync})
if [ ${synce} = true ]; then
echo "Read log related to local folder ${localsync}"
grep ${localsync} $SYNCFILE | while read hash pid when status localpath remotepath; do
readlog ${hash}
done
else
echo "local folder ${localsync} not in sync task, you can not read the log"
fi
else
echo "ATTENTION $localsync not exists Aborting log!"
exit 2
fi
;;
s) indexsync=${OPTARG}
empty=$(emptysyncfile)
if [ ${empty} = true ]; then
echo "Sync ${indexsync} not exists, sync file empty, please use init first"
else
re_num='^[0-9]+$'
if [[ ! $indexsync =~ $re_num ]]; then
echo -e "-s option must be ${RED}integer${ENDCOLOR}"
exit 3
fi
oi=$(syncoutofindex $indexsync)
if [ ${oi} = true ]; then
echo "Sync ${indexsync} out of index"
exit 1
else
sed -n -s "$indexsync"p "$SYNCFILE" | while read hash pid when status localpath remotepath; do
readlog ${hash}
done
fi
fi
;;
\?) myhelp
exit 1
;;
esac
done
;;
2021-06-09 17:04:43 +02:00
stop)
shift
while getopts ":l:s:" option; do
case "${option}" in
l) localsync=${OPTARG}
if [[ -d "$localsync" ]]; then
synce=$(syncexists ${localsync})
if [ ${synce} = true ]; then
stoploopsyncpath ${localsync}
else
2021-06-12 22:33:52 +02:00
echo -e "Can't stop local sync for folder ${RED}${localsync}${ENDCOLOR}. It is not in sync task, please before use init"
2021-06-09 17:04:43 +02:00
exit 2
fi
else
2021-06-12 22:33:52 +02:00
echo -e "Local folder ${RED}${localsync}${ENDCOLOR} not exists Aborting!"
2021-06-09 17:04:43 +02:00
exit 2
fi
;;
s) indexsync=${OPTARG}
empty=$(emptysyncfile)
if [ ${empty} = true ]; then
2021-06-12 22:33:52 +02:00
echo "Sync file empty, please use init first for creating a sync"
exit 2
2021-06-09 17:04:43 +02:00
else
2021-06-12 22:33:52 +02:00
re_num='^[0-9]+$'
if [[ ! $indexsync =~ $re_num ]]; then
echo -e "-s option must be ${RED}integer${ENDCOLOR}"
exit 3
fi
2021-06-09 17:04:43 +02:00
oi=$(syncoutofindex $indexsync)
if [ ${oi} = true ]; then
2021-06-12 22:33:52 +02:00
echo -e "${RED}Sync out of index${ENDCOLOR}"
exit 2
2021-06-09 17:04:43 +02:00
else
stoploopsyncpath $indexsync
fi
fi
;;
\?) myhelp
exit 1
;;
esac
done
;;
list)
if [ ! -f $SYNCFILE ]; then
echo 'Sync file empty'
else
if [ $(cat $SYNCFILE | wc -l) -eq 0 ]; then
echo 'Sync file empty'
else
echo -e "$(cat $SYNCFILE | cat -n | sed -e "s/RUNNING/\\${GREEN}RUNNING\\${ENDCOLOR}/" -e "s/STOPPED/\\${RED}STOPPED\\${ENDCOLOR}/")"
fi
fi
;;
remove)
shift
while getopts ":l:s:" option; do
case "${option}" in
l) localsync=${OPTARG}
#echo "Sync this local folder: $2"
synce=$(syncexists ${localsync})
if [ ${synce} = true ]; then
echo "Removing Sync local folder ${localsync}"
removeloopsyncpath ${localsync}
else
echo "Can't remove local sync for folder ${localsync}. It is not in sync task"
exit 2
fi
;;
s) indexsync=${OPTARG}
empty=$(emptysyncfile)
if [ ${empty} = true ]; then
echo "Sync ${indexsync} not exists, sync file empty, please use init first"
else
oi=$(syncoutofindex $indexsync)
if [ ${oi} = true ]; then
echo "Sync ${indexsync} out of index"
exit 1
else
removeloopsyncpath $indexsync
fi
fi
;;
\?) myhelp
exit 1
;;
esac
done
;;
*)
myhelp
exit 1
esac