notanamber/masy
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

centralize the generation of files

Browse Source
This commit is contained in:
Amber 2022-11-07 21:45:04 +01:00
parent 6729906e5a
commit 9463c0b059
4 changed files with 72 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
.filetemplates.sh Normal file
View File

@ -0,0 +1,42 @@
#!/bin/bash
# Copyright 2021 Luca Paris
#This file is part of masync.
#masync is free software: you can redistribute it and/or modify
#it under the terms of the GNU General Public License as published by
#the Free Software Foundation, either version 3 of the License, or
#(at your option) any later version.
#masync is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#GNU General Public License for more details.
#You should have received a copy of the GNU General Public License
#along with masync. If not, see <http://www.gnu.org/licenses/>.
#
# the file contains all sync managed by masync
#
SYNCFILE=~/.syncdir.sync
#
# the file contains the snapshot of a stopped sync
#
SNAPSHOTFILE=~/.snapshot_
TMPQUEUEFILE=~/.syncdir_{{hash}}.queue
TMPQUEUEDELETES=~/.syncdir_{{hash}}.deletes
################################
# $1 -> instr, the string template to format, we expect it contains placeholders {{placeholder_name}}
# $[2..n] -> placeholder in the form placeholder_name=value
################################
format() {
instr=$1
shift
for arg in "$@"; do
IFS== read -r key val <<< $arg
instr=$(echo $instr | sed -e "s/{{$key}}/$val/g")
done
echo $instr
}

1
.synccmd.sh
View File

@ -29,6 +29,7 @@ sync() {
# PUSH TO REMOTE - and PULL FROM REMOTE swapping `src` with `dest`
#echo "executing rsync -aPu --log-file=$LOGFILE -e ssh $otheropts $src $dest"
# NOTE REMOTEDIR ALREADY EXISTS IN DESTINATION
# rsync -ni -aPu --progress --out-format="%i %n%L %''b" -e ssh notanamber@myvps:/home/notanamber/notes/ /home/luca/sharednotes/
rsync -aPu --log-file=$LOGFILE -e ssh $otheropts $src $dest 1>/dev/null 2>&1
# last command execution exit code $?"
rsync_result=$?

8
masync
View File

@ -16,9 +16,8 @@
#along with masync. If not, see <http://www.gnu.org/licenses/>.
source ~/bin/.synccmd.sh
source ~/bin/.filetemplates.sh
source ~/bin/.colordef.sh
SYNCFILE=~/.syncdir.sync
SNAPSHOTFILE=~/.snapshot_
MYPID=$$
STATUS_RUNNING=RUNNING
STATUS_STOPPED=STOPPED
@ -78,7 +77,7 @@ syncexists() {
}
feeddeletes() {
queuedeletes=~/.syncdir_$1.deletes
queuedeletes=$(format ${TMPQUEUEDELETES} hash=$1)
snapfile=${SNAPSHOTFILE}$1
grep $1 ${SYNCFILE} | while read hash pid when status localpath remotepath; do
remoterelativepath=$(echo "$remotepath" | cut -d : -f 2)
@ -141,7 +140,8 @@ startsyncpath() {
echo -e "Start synching local folder ${GREEN}$1${ENDCOLOR} with remote ${GREEN}$2${ENDCOLOR}..."
localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -)
snapshotfile=${SNAPSHOTFILE}${localpath_hash}
tempqueuefile=~/.syncdir_${localpath_hash}.queue
#tempqueuefile=~/.syncdir_${localpath_hash}.queue
tempqueuefile=$(format ${TMPQUEUEFILE} hash=${localpath_hash})
if [ -e ${snapshotfile} ]; then
echo "Founded snapshot file: ${snapshotfile} collect files to delete in origin"

49
syncloop.sh
View File

@ -21,10 +21,11 @@
# @2 local path to sync
# @3 remote path to sync
#echo "$1, $2, $3"
source ~/bin/.filetemplates.sh
source ~/bin/.synccmd.sh
source ~/bin/.colordef.sh
TMPQUEUEFILE=~/.syncdir_$1.queue
TMPQUEUEDELETES=~/.syncdir_$1.deletes
tmpqueuefile=$(format ${TMPQUEUEFILE} hash=$1)
tmpqueuedeletes=$(format ${TMPQUEUEDELETES} hash=$1)
LOCALPATH_HASH=$1
LOCALPATH=$2
@ -34,9 +35,9 @@ REMOTEHOST=$(echo "$REMOTEPATH" | cut -d : -f 1 | cut -d @ -f 2)
REMOTERELATIVEPATH=$(echo "$REMOTEPATH" | cut -d : -f 2)
#acquire lock_deletes file descritptor
exec {lock_deletes}>>${TMPQUEUEDELETES}
exec {lock_deletes}>>${tmpqueuedeletes}
#acquire lock queue file descriptor
exec {lock_queue}>>${TMPQUEUEFILE}
exec {lock_queue}>>${tmpqueuefile}
min() {
if [ $1 -gt $2 ]; then
@ -58,42 +59,42 @@ synccycle() {
thresholdpulllimit=600
while :
do
if [ -f "$TMPQUEUEFILE" ] || [ -f "$TMPQUEUEDELETES" ]; then
if [ -f "$tmpqueuefile" ] || [ -f "$tmpqueuedeletes" ]; then
### DELETES WHILE LOOP
if [ -f "$TMPQUEUEDELETES" ]; then
ndeletes=$(wc -l < ${TMPQUEUEDELETES})
if [ -f "$tmpqueuedeletes" ]; then
ndeletes=$(wc -l < ${tmpqueuedeletes})
while [ $ndeletes -gt 0 ]; do
echo -n -e "[Masync log - PUSH FILES TO DELETE] to remote: ${RED}$(head -n $ndeletes ${TMPQUEUEDELETES} | tr '\n', ' ')${ENDCOLOR}"
ssh $REMOTEHOST "rm -rf $(head -n $ndeletes ${TMPQUEUEDELETES} | tr '\n', ' ')"
echo -n -e "[Masync log - PUSH FILES TO DELETE] to remote: ${RED}$(head -n $ndeletes ${tmpqueuedeletes} | tr '\n', ' ')${ENDCOLOR}"
ssh $REMOTEHOST "rm -rf $(head -n $ndeletes ${tmpqueuedeletes} | tr '\n', ' ')"
echo -e " ${PURPLE}DONE${ENDCOLOR}"
# remove the first ndeletes lines
flock $lock_deletes
tmpfile=$(mktemp)
tail -n +$(expr ${ndeletes} + 1) < ${TMPQUEUEDELETES} > ${tmpfile}
cat ${tmpfile} > ${TMPQUEUEDELETES}
tail -n +$(expr ${ndeletes} + 1) < ${tmpqueuedeletes} > ${tmpfile}
cat ${tmpfile} > ${tmpqueuedeletes}
rm -f ${tmpfile}
flock -u $lock_deletes
ndeletes=$(wc -l < ${TMPQUEUEDELETES})
ndeletes=$(wc -l < ${tmpqueuedeletes})
done
fi
### PUSH WHILE LOOP
if [ -f "$TMPQUEUEFILE" ]; then
nqueue=$(wc -l < ${TMPQUEUEFILE})
if [ -f "$tmpqueuefile" ]; then
nqueue=$(wc -l < ${tmpqueuefile})
while [ $nqueue -gt 0 ]; do
#echo "${TMPQUEUEFILE} not empty consume ${nqueue} updates in queue"
echo -n -e "[Masync log - PUSH FILES] to remote: ${GREEN}$(head -n ${nqueue} ${TMPQUEUEFILE} | tr '\n', ' ')${ENDCOLOR}"
#echo "${tmpqueuefile} not empty consume ${nqueue} updates in queue"
echo -n -e "[Masync log - PUSH FILES] to remote: ${GREEN}$(head -n ${nqueue} ${tmpqueuefile} | tr '\n', ' ')${ENDCOLOR}"
sync $LOCALPATH_HASH $LOCALPATH $REMOTEPATH
echo -e " ${PURPLE}DONE${ENDCOLOR}"
# remove the first nqueue lines from queue
flock $lock_queue
tmpfile=$(mktemp)
tail -n +$(expr ${nqueue} + 1) < ${TMPQUEUEFILE} > ${tmpfile}
cat ${tmpfile} > ${TMPQUEUEFILE}
tail -n +$(expr ${nqueue} + 1) < ${tmpqueuefile} > ${tmpfile}
cat ${tmpfile} > ${tmpqueuefile}
rm -f ${tmpfile}
flock -u $lock_queue
#lastsynctime="$(date -u +%s.%N)"
nqueue=$(wc -l < ${TMPQUEUEFILE})
nqueue=$(wc -l < ${tmpqueuefile})
done
fi
fi
@ -125,17 +126,17 @@ inotifywait -m -r -e create -e close_write -e move -e delete --exclude "\.swp|\.
if [ $action = 'DELETE' ] || [ $action = 'DELETE,ISDIR' ] || [ $action = 'MOVED_FROM' ] || [ $action = 'MOVED_FROM,ISDIR' ]; then
echo -e "[Masync log - ENQUEUE delete] $dir$file"
flock $lock_deletes
echo $dir$file | sed -e "s~$LOCALPATH~$REMOTERELATIVEPATH~g" | tee -a ${TMPQUEUEDELETES} 1>/dev/null
echo $dir$file | sed -e "s~$LOCALPATH~$REMOTERELATIVEPATH~g" | tee -a ${tmpqueuedeletes} 1>/dev/null
flock -u $lock_deletes
else
flock $lock_queue
## add to queue only if not already in it
if [ $(grep -E ${file}$ ${TMPQUEUEFILE} | wc -l) -lt 1 ]; then
#echo "Add notify to queue: file '$file' in directory '$dir' for '$action'" | tee -a ${TMPQUEUEFILE}
if [ $(grep -E ${file}$ ${tmpqueuefile} | wc -l) -lt 1 ]; then
#echo "Add notify to queue: file '$file' in directory '$dir' for '$action'" | tee -a ${tmpqueuefile}
echo -e "[Masync log - ENQUEUE file] $dir$file"
# AVOID TO USE TEE: SIMPLY append
#echo ${dir}${file} | tee -a ${TMPQUEUEFILE}
echo ${dir}${file} >> ${TMPQUEUEFILE}
#echo ${dir}${file} | tee -a ${tmpqueuefile}
echo ${dir}${file} >> ${tmpqueuefile}
fi
flock -u $lock_queue
fi