From 9463c0b0591171e9a443794d12c502e3ac9a596c Mon Sep 17 00:00:00 2001 From: Amber Date: Mon, 7 Nov 2022 21:45:04 +0100 Subject: [PATCH] centralize the generation of files --- .filetemplates.sh | 42 ++++++++++++++++++++++++++++++++++++++++ .synccmd.sh | 1 + masync | 8 ++++---- syncloop.sh | 49 ++++++++++++++++++++++++----------------------- 4 files changed, 72 insertions(+), 28 deletions(-) create mode 100644 .filetemplates.sh diff --git a/.filetemplates.sh b/.filetemplates.sh new file mode 100644 index 0000000..4ef530a --- /dev/null +++ b/.filetemplates.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Copyright 2021 Luca Paris +#This file is part of masync. + +#masync is free software: you can redistribute it and/or modify +#it under the terms of the GNU General Public License as published by +#the Free Software Foundation, either version 3 of the License, or +#(at your option) any later version. + +#masync is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#GNU General Public License for more details. + +#You should have received a copy of the GNU General Public License +#along with masync. If not, see . + +# +# the file contains all sync managed by masync +# +SYNCFILE=~/.syncdir.sync +# +# the file contains the snapshot of a stopped sync +# +SNAPSHOTFILE=~/.snapshot_ +TMPQUEUEFILE=~/.syncdir_{{hash}}.queue +TMPQUEUEDELETES=~/.syncdir_{{hash}}.deletes + +################################ +# $1 -> instr, the string template to format, we expect it contains placeholders {{placeholder_name}} +# $[2..n] -> placeholder in the form placeholder_name=value +################################ + +format() { + instr=$1 + shift + for arg in "$@"; do + IFS== read -r key val <<< $arg + instr=$(echo $instr | sed -e "s/{{$key}}/$val/g") + done + echo $instr +} diff --git a/.synccmd.sh b/.synccmd.sh index 63a875d..471b27b 100755 --- a/.synccmd.sh +++ b/.synccmd.sh @@ -29,6 +29,7 @@ sync() { # PUSH TO REMOTE - and PULL FROM REMOTE swapping `src` with `dest` #echo "executing rsync -aPu --log-file=$LOGFILE -e ssh $otheropts $src $dest" # NOTE REMOTEDIR ALREADY EXISTS IN DESTINATION + # rsync -ni -aPu --progress --out-format="%i %n%L %''b" -e ssh notanamber@myvps:/home/notanamber/notes/ /home/luca/sharednotes/ rsync -aPu --log-file=$LOGFILE -e ssh $otheropts $src $dest 1>/dev/null 2>&1 # last command execution exit code $?" rsync_result=$? diff --git a/masync b/masync index e33a028..6f6675e 100755 --- a/masync +++ b/masync @@ -16,9 +16,8 @@ #along with masync. If not, see . source ~/bin/.synccmd.sh +source ~/bin/.filetemplates.sh source ~/bin/.colordef.sh -SYNCFILE=~/.syncdir.sync -SNAPSHOTFILE=~/.snapshot_ MYPID=$$ STATUS_RUNNING=RUNNING STATUS_STOPPED=STOPPED @@ -78,7 +77,7 @@ syncexists() { } feeddeletes() { - queuedeletes=~/.syncdir_$1.deletes + queuedeletes=$(format ${TMPQUEUEDELETES} hash=$1) snapfile=${SNAPSHOTFILE}$1 grep $1 ${SYNCFILE} | while read hash pid when status localpath remotepath; do remoterelativepath=$(echo "$remotepath" | cut -d : -f 2) @@ -141,7 +140,8 @@ startsyncpath() { echo -e "Start synching local folder ${GREEN}$1${ENDCOLOR} with remote ${GREEN}$2${ENDCOLOR}..." localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -) snapshotfile=${SNAPSHOTFILE}${localpath_hash} - tempqueuefile=~/.syncdir_${localpath_hash}.queue + #tempqueuefile=~/.syncdir_${localpath_hash}.queue + tempqueuefile=$(format ${TMPQUEUEFILE} hash=${localpath_hash}) if [ -e ${snapshotfile} ]; then echo "Founded snapshot file: ${snapshotfile} collect files to delete in origin" diff --git a/syncloop.sh b/syncloop.sh index 083e00b..0c220aa 100755 --- a/syncloop.sh +++ b/syncloop.sh @@ -21,10 +21,11 @@ # @2 local path to sync # @3 remote path to sync #echo "$1, $2, $3" +source ~/bin/.filetemplates.sh source ~/bin/.synccmd.sh source ~/bin/.colordef.sh -TMPQUEUEFILE=~/.syncdir_$1.queue -TMPQUEUEDELETES=~/.syncdir_$1.deletes +tmpqueuefile=$(format ${TMPQUEUEFILE} hash=$1) +tmpqueuedeletes=$(format ${TMPQUEUEDELETES} hash=$1) LOCALPATH_HASH=$1 LOCALPATH=$2 @@ -34,9 +35,9 @@ REMOTEHOST=$(echo "$REMOTEPATH" | cut -d : -f 1 | cut -d @ -f 2) REMOTERELATIVEPATH=$(echo "$REMOTEPATH" | cut -d : -f 2) #acquire lock_deletes file descritptor -exec {lock_deletes}>>${TMPQUEUEDELETES} +exec {lock_deletes}>>${tmpqueuedeletes} #acquire lock queue file descriptor -exec {lock_queue}>>${TMPQUEUEFILE} +exec {lock_queue}>>${tmpqueuefile} min() { if [ $1 -gt $2 ]; then @@ -58,42 +59,42 @@ synccycle() { thresholdpulllimit=600 while : do - if [ -f "$TMPQUEUEFILE" ] || [ -f "$TMPQUEUEDELETES" ]; then + if [ -f "$tmpqueuefile" ] || [ -f "$tmpqueuedeletes" ]; then ### DELETES WHILE LOOP - if [ -f "$TMPQUEUEDELETES" ]; then - ndeletes=$(wc -l < ${TMPQUEUEDELETES}) + if [ -f "$tmpqueuedeletes" ]; then + ndeletes=$(wc -l < ${tmpqueuedeletes}) while [ $ndeletes -gt 0 ]; do - echo -n -e "[Masync log - PUSH FILES TO DELETE] to remote: ${RED}$(head -n $ndeletes ${TMPQUEUEDELETES} | tr '\n', ' ')${ENDCOLOR}" - ssh $REMOTEHOST "rm -rf $(head -n $ndeletes ${TMPQUEUEDELETES} | tr '\n', ' ')" + echo -n -e "[Masync log - PUSH FILES TO DELETE] to remote: ${RED}$(head -n $ndeletes ${tmpqueuedeletes} | tr '\n', ' ')${ENDCOLOR}" + ssh $REMOTEHOST "rm -rf $(head -n $ndeletes ${tmpqueuedeletes} | tr '\n', ' ')" echo -e " ${PURPLE}DONE${ENDCOLOR}" # remove the first ndeletes lines flock $lock_deletes tmpfile=$(mktemp) - tail -n +$(expr ${ndeletes} + 1) < ${TMPQUEUEDELETES} > ${tmpfile} - cat ${tmpfile} > ${TMPQUEUEDELETES} + tail -n +$(expr ${ndeletes} + 1) < ${tmpqueuedeletes} > ${tmpfile} + cat ${tmpfile} > ${tmpqueuedeletes} rm -f ${tmpfile} flock -u $lock_deletes - ndeletes=$(wc -l < ${TMPQUEUEDELETES}) + ndeletes=$(wc -l < ${tmpqueuedeletes}) done fi ### PUSH WHILE LOOP - if [ -f "$TMPQUEUEFILE" ]; then - nqueue=$(wc -l < ${TMPQUEUEFILE}) + if [ -f "$tmpqueuefile" ]; then + nqueue=$(wc -l < ${tmpqueuefile}) while [ $nqueue -gt 0 ]; do - #echo "${TMPQUEUEFILE} not empty consume ${nqueue} updates in queue" - echo -n -e "[Masync log - PUSH FILES] to remote: ${GREEN}$(head -n ${nqueue} ${TMPQUEUEFILE} | tr '\n', ' ')${ENDCOLOR}" + #echo "${tmpqueuefile} not empty consume ${nqueue} updates in queue" + echo -n -e "[Masync log - PUSH FILES] to remote: ${GREEN}$(head -n ${nqueue} ${tmpqueuefile} | tr '\n', ' ')${ENDCOLOR}" sync $LOCALPATH_HASH $LOCALPATH $REMOTEPATH echo -e " ${PURPLE}DONE${ENDCOLOR}" # remove the first nqueue lines from queue flock $lock_queue tmpfile=$(mktemp) - tail -n +$(expr ${nqueue} + 1) < ${TMPQUEUEFILE} > ${tmpfile} - cat ${tmpfile} > ${TMPQUEUEFILE} + tail -n +$(expr ${nqueue} + 1) < ${tmpqueuefile} > ${tmpfile} + cat ${tmpfile} > ${tmpqueuefile} rm -f ${tmpfile} flock -u $lock_queue #lastsynctime="$(date -u +%s.%N)" - nqueue=$(wc -l < ${TMPQUEUEFILE}) + nqueue=$(wc -l < ${tmpqueuefile}) done fi fi @@ -125,17 +126,17 @@ inotifywait -m -r -e create -e close_write -e move -e delete --exclude "\.swp|\. if [ $action = 'DELETE' ] || [ $action = 'DELETE,ISDIR' ] || [ $action = 'MOVED_FROM' ] || [ $action = 'MOVED_FROM,ISDIR' ]; then echo -e "[Masync log - ENQUEUE delete] $dir$file" flock $lock_deletes - echo $dir$file | sed -e "s~$LOCALPATH~$REMOTERELATIVEPATH~g" | tee -a ${TMPQUEUEDELETES} 1>/dev/null + echo $dir$file | sed -e "s~$LOCALPATH~$REMOTERELATIVEPATH~g" | tee -a ${tmpqueuedeletes} 1>/dev/null flock -u $lock_deletes else flock $lock_queue ## add to queue only if not already in it - if [ $(grep -E ${file}$ ${TMPQUEUEFILE} | wc -l) -lt 1 ]; then - #echo "Add notify to queue: file '$file' in directory '$dir' for '$action'" | tee -a ${TMPQUEUEFILE} + if [ $(grep -E ${file}$ ${tmpqueuefile} | wc -l) -lt 1 ]; then + #echo "Add notify to queue: file '$file' in directory '$dir' for '$action'" | tee -a ${tmpqueuefile} echo -e "[Masync log - ENQUEUE file] $dir$file" # AVOID TO USE TEE: SIMPLY append - #echo ${dir}${file} | tee -a ${TMPQUEUEFILE} - echo ${dir}${file} >> ${TMPQUEUEFILE} + #echo ${dir}${file} | tee -a ${tmpqueuefile} + echo ${dir}${file} >> ${tmpqueuefile} fi flock -u $lock_queue fi