centralize the generation of files
This commit is contained in:
parent
6729906e5a
commit
9463c0b059
|
@ -0,0 +1,42 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# Copyright 2021 Luca Paris
|
||||||
|
#This file is part of masync.
|
||||||
|
|
||||||
|
#masync is free software: you can redistribute it and/or modify
|
||||||
|
#it under the terms of the GNU General Public License as published by
|
||||||
|
#the Free Software Foundation, either version 3 of the License, or
|
||||||
|
#(at your option) any later version.
|
||||||
|
|
||||||
|
#masync is distributed in the hope that it will be useful,
|
||||||
|
#but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
#GNU General Public License for more details.
|
||||||
|
|
||||||
|
#You should have received a copy of the GNU General Public License
|
||||||
|
#along with masync. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
#
|
||||||
|
# the file contains all sync managed by masync
|
||||||
|
#
|
||||||
|
SYNCFILE=~/.syncdir.sync
|
||||||
|
#
|
||||||
|
# the file contains the snapshot of a stopped sync
|
||||||
|
#
|
||||||
|
SNAPSHOTFILE=~/.snapshot_
|
||||||
|
TMPQUEUEFILE=~/.syncdir_{{hash}}.queue
|
||||||
|
TMPQUEUEDELETES=~/.syncdir_{{hash}}.deletes
|
||||||
|
|
||||||
|
################################
|
||||||
|
# $1 -> instr, the string template to format, we expect it contains placeholders {{placeholder_name}}
|
||||||
|
# $[2..n] -> placeholder in the form placeholder_name=value
|
||||||
|
################################
|
||||||
|
|
||||||
|
format() {
|
||||||
|
instr=$1
|
||||||
|
shift
|
||||||
|
for arg in "$@"; do
|
||||||
|
IFS== read -r key val <<< $arg
|
||||||
|
instr=$(echo $instr | sed -e "s/{{$key}}/$val/g")
|
||||||
|
done
|
||||||
|
echo $instr
|
||||||
|
}
|
|
@ -29,6 +29,7 @@ sync() {
|
||||||
# PUSH TO REMOTE - and PULL FROM REMOTE swapping `src` with `dest`
|
# PUSH TO REMOTE - and PULL FROM REMOTE swapping `src` with `dest`
|
||||||
#echo "executing rsync -aPu --log-file=$LOGFILE -e ssh $otheropts $src $dest"
|
#echo "executing rsync -aPu --log-file=$LOGFILE -e ssh $otheropts $src $dest"
|
||||||
# NOTE REMOTEDIR ALREADY EXISTS IN DESTINATION
|
# NOTE REMOTEDIR ALREADY EXISTS IN DESTINATION
|
||||||
|
# rsync -ni -aPu --progress --out-format="%i %n%L %''b" -e ssh notanamber@myvps:/home/notanamber/notes/ /home/luca/sharednotes/
|
||||||
rsync -aPu --log-file=$LOGFILE -e ssh $otheropts $src $dest 1>/dev/null 2>&1
|
rsync -aPu --log-file=$LOGFILE -e ssh $otheropts $src $dest 1>/dev/null 2>&1
|
||||||
# last command execution exit code $?"
|
# last command execution exit code $?"
|
||||||
rsync_result=$?
|
rsync_result=$?
|
||||||
|
|
8
masync
8
masync
|
@ -16,9 +16,8 @@
|
||||||
#along with masync. If not, see <http://www.gnu.org/licenses/>.
|
#along with masync. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
source ~/bin/.synccmd.sh
|
source ~/bin/.synccmd.sh
|
||||||
|
source ~/bin/.filetemplates.sh
|
||||||
source ~/bin/.colordef.sh
|
source ~/bin/.colordef.sh
|
||||||
SYNCFILE=~/.syncdir.sync
|
|
||||||
SNAPSHOTFILE=~/.snapshot_
|
|
||||||
MYPID=$$
|
MYPID=$$
|
||||||
STATUS_RUNNING=RUNNING
|
STATUS_RUNNING=RUNNING
|
||||||
STATUS_STOPPED=STOPPED
|
STATUS_STOPPED=STOPPED
|
||||||
|
@ -78,7 +77,7 @@ syncexists() {
|
||||||
}
|
}
|
||||||
|
|
||||||
feeddeletes() {
|
feeddeletes() {
|
||||||
queuedeletes=~/.syncdir_$1.deletes
|
queuedeletes=$(format ${TMPQUEUEDELETES} hash=$1)
|
||||||
snapfile=${SNAPSHOTFILE}$1
|
snapfile=${SNAPSHOTFILE}$1
|
||||||
grep $1 ${SYNCFILE} | while read hash pid when status localpath remotepath; do
|
grep $1 ${SYNCFILE} | while read hash pid when status localpath remotepath; do
|
||||||
remoterelativepath=$(echo "$remotepath" | cut -d : -f 2)
|
remoterelativepath=$(echo "$remotepath" | cut -d : -f 2)
|
||||||
|
@ -141,7 +140,8 @@ startsyncpath() {
|
||||||
echo -e "Start synching local folder ${GREEN}$1${ENDCOLOR} with remote ${GREEN}$2${ENDCOLOR}..."
|
echo -e "Start synching local folder ${GREEN}$1${ENDCOLOR} with remote ${GREEN}$2${ENDCOLOR}..."
|
||||||
localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -)
|
localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -)
|
||||||
snapshotfile=${SNAPSHOTFILE}${localpath_hash}
|
snapshotfile=${SNAPSHOTFILE}${localpath_hash}
|
||||||
tempqueuefile=~/.syncdir_${localpath_hash}.queue
|
#tempqueuefile=~/.syncdir_${localpath_hash}.queue
|
||||||
|
tempqueuefile=$(format ${TMPQUEUEFILE} hash=${localpath_hash})
|
||||||
|
|
||||||
if [ -e ${snapshotfile} ]; then
|
if [ -e ${snapshotfile} ]; then
|
||||||
echo "Founded snapshot file: ${snapshotfile} collect files to delete in origin"
|
echo "Founded snapshot file: ${snapshotfile} collect files to delete in origin"
|
||||||
|
|
49
syncloop.sh
49
syncloop.sh
|
@ -21,10 +21,11 @@
|
||||||
# @2 local path to sync
|
# @2 local path to sync
|
||||||
# @3 remote path to sync
|
# @3 remote path to sync
|
||||||
#echo "$1, $2, $3"
|
#echo "$1, $2, $3"
|
||||||
|
source ~/bin/.filetemplates.sh
|
||||||
source ~/bin/.synccmd.sh
|
source ~/bin/.synccmd.sh
|
||||||
source ~/bin/.colordef.sh
|
source ~/bin/.colordef.sh
|
||||||
TMPQUEUEFILE=~/.syncdir_$1.queue
|
tmpqueuefile=$(format ${TMPQUEUEFILE} hash=$1)
|
||||||
TMPQUEUEDELETES=~/.syncdir_$1.deletes
|
tmpqueuedeletes=$(format ${TMPQUEUEDELETES} hash=$1)
|
||||||
|
|
||||||
LOCALPATH_HASH=$1
|
LOCALPATH_HASH=$1
|
||||||
LOCALPATH=$2
|
LOCALPATH=$2
|
||||||
|
@ -34,9 +35,9 @@ REMOTEHOST=$(echo "$REMOTEPATH" | cut -d : -f 1 | cut -d @ -f 2)
|
||||||
REMOTERELATIVEPATH=$(echo "$REMOTEPATH" | cut -d : -f 2)
|
REMOTERELATIVEPATH=$(echo "$REMOTEPATH" | cut -d : -f 2)
|
||||||
|
|
||||||
#acquire lock_deletes file descritptor
|
#acquire lock_deletes file descritptor
|
||||||
exec {lock_deletes}>>${TMPQUEUEDELETES}
|
exec {lock_deletes}>>${tmpqueuedeletes}
|
||||||
#acquire lock queue file descriptor
|
#acquire lock queue file descriptor
|
||||||
exec {lock_queue}>>${TMPQUEUEFILE}
|
exec {lock_queue}>>${tmpqueuefile}
|
||||||
|
|
||||||
min() {
|
min() {
|
||||||
if [ $1 -gt $2 ]; then
|
if [ $1 -gt $2 ]; then
|
||||||
|
@ -58,42 +59,42 @@ synccycle() {
|
||||||
thresholdpulllimit=600
|
thresholdpulllimit=600
|
||||||
while :
|
while :
|
||||||
do
|
do
|
||||||
if [ -f "$TMPQUEUEFILE" ] || [ -f "$TMPQUEUEDELETES" ]; then
|
if [ -f "$tmpqueuefile" ] || [ -f "$tmpqueuedeletes" ]; then
|
||||||
### DELETES WHILE LOOP
|
### DELETES WHILE LOOP
|
||||||
if [ -f "$TMPQUEUEDELETES" ]; then
|
if [ -f "$tmpqueuedeletes" ]; then
|
||||||
ndeletes=$(wc -l < ${TMPQUEUEDELETES})
|
ndeletes=$(wc -l < ${tmpqueuedeletes})
|
||||||
while [ $ndeletes -gt 0 ]; do
|
while [ $ndeletes -gt 0 ]; do
|
||||||
echo -n -e "[Masync log - PUSH FILES TO DELETE] to remote: ${RED}$(head -n $ndeletes ${TMPQUEUEDELETES} | tr '\n', ' ')${ENDCOLOR}"
|
echo -n -e "[Masync log - PUSH FILES TO DELETE] to remote: ${RED}$(head -n $ndeletes ${tmpqueuedeletes} | tr '\n', ' ')${ENDCOLOR}"
|
||||||
ssh $REMOTEHOST "rm -rf $(head -n $ndeletes ${TMPQUEUEDELETES} | tr '\n', ' ')"
|
ssh $REMOTEHOST "rm -rf $(head -n $ndeletes ${tmpqueuedeletes} | tr '\n', ' ')"
|
||||||
echo -e " ${PURPLE}DONE${ENDCOLOR}"
|
echo -e " ${PURPLE}DONE${ENDCOLOR}"
|
||||||
# remove the first ndeletes lines
|
# remove the first ndeletes lines
|
||||||
flock $lock_deletes
|
flock $lock_deletes
|
||||||
tmpfile=$(mktemp)
|
tmpfile=$(mktemp)
|
||||||
tail -n +$(expr ${ndeletes} + 1) < ${TMPQUEUEDELETES} > ${tmpfile}
|
tail -n +$(expr ${ndeletes} + 1) < ${tmpqueuedeletes} > ${tmpfile}
|
||||||
cat ${tmpfile} > ${TMPQUEUEDELETES}
|
cat ${tmpfile} > ${tmpqueuedeletes}
|
||||||
rm -f ${tmpfile}
|
rm -f ${tmpfile}
|
||||||
flock -u $lock_deletes
|
flock -u $lock_deletes
|
||||||
ndeletes=$(wc -l < ${TMPQUEUEDELETES})
|
ndeletes=$(wc -l < ${tmpqueuedeletes})
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
### PUSH WHILE LOOP
|
### PUSH WHILE LOOP
|
||||||
if [ -f "$TMPQUEUEFILE" ]; then
|
if [ -f "$tmpqueuefile" ]; then
|
||||||
nqueue=$(wc -l < ${TMPQUEUEFILE})
|
nqueue=$(wc -l < ${tmpqueuefile})
|
||||||
while [ $nqueue -gt 0 ]; do
|
while [ $nqueue -gt 0 ]; do
|
||||||
#echo "${TMPQUEUEFILE} not empty consume ${nqueue} updates in queue"
|
#echo "${tmpqueuefile} not empty consume ${nqueue} updates in queue"
|
||||||
echo -n -e "[Masync log - PUSH FILES] to remote: ${GREEN}$(head -n ${nqueue} ${TMPQUEUEFILE} | tr '\n', ' ')${ENDCOLOR}"
|
echo -n -e "[Masync log - PUSH FILES] to remote: ${GREEN}$(head -n ${nqueue} ${tmpqueuefile} | tr '\n', ' ')${ENDCOLOR}"
|
||||||
sync $LOCALPATH_HASH $LOCALPATH $REMOTEPATH
|
sync $LOCALPATH_HASH $LOCALPATH $REMOTEPATH
|
||||||
echo -e " ${PURPLE}DONE${ENDCOLOR}"
|
echo -e " ${PURPLE}DONE${ENDCOLOR}"
|
||||||
# remove the first nqueue lines from queue
|
# remove the first nqueue lines from queue
|
||||||
flock $lock_queue
|
flock $lock_queue
|
||||||
tmpfile=$(mktemp)
|
tmpfile=$(mktemp)
|
||||||
tail -n +$(expr ${nqueue} + 1) < ${TMPQUEUEFILE} > ${tmpfile}
|
tail -n +$(expr ${nqueue} + 1) < ${tmpqueuefile} > ${tmpfile}
|
||||||
cat ${tmpfile} > ${TMPQUEUEFILE}
|
cat ${tmpfile} > ${tmpqueuefile}
|
||||||
rm -f ${tmpfile}
|
rm -f ${tmpfile}
|
||||||
flock -u $lock_queue
|
flock -u $lock_queue
|
||||||
#lastsynctime="$(date -u +%s.%N)"
|
#lastsynctime="$(date -u +%s.%N)"
|
||||||
nqueue=$(wc -l < ${TMPQUEUEFILE})
|
nqueue=$(wc -l < ${tmpqueuefile})
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
@ -125,17 +126,17 @@ inotifywait -m -r -e create -e close_write -e move -e delete --exclude "\.swp|\.
|
||||||
if [ $action = 'DELETE' ] || [ $action = 'DELETE,ISDIR' ] || [ $action = 'MOVED_FROM' ] || [ $action = 'MOVED_FROM,ISDIR' ]; then
|
if [ $action = 'DELETE' ] || [ $action = 'DELETE,ISDIR' ] || [ $action = 'MOVED_FROM' ] || [ $action = 'MOVED_FROM,ISDIR' ]; then
|
||||||
echo -e "[Masync log - ENQUEUE delete] $dir$file"
|
echo -e "[Masync log - ENQUEUE delete] $dir$file"
|
||||||
flock $lock_deletes
|
flock $lock_deletes
|
||||||
echo $dir$file | sed -e "s~$LOCALPATH~$REMOTERELATIVEPATH~g" | tee -a ${TMPQUEUEDELETES} 1>/dev/null
|
echo $dir$file | sed -e "s~$LOCALPATH~$REMOTERELATIVEPATH~g" | tee -a ${tmpqueuedeletes} 1>/dev/null
|
||||||
flock -u $lock_deletes
|
flock -u $lock_deletes
|
||||||
else
|
else
|
||||||
flock $lock_queue
|
flock $lock_queue
|
||||||
## add to queue only if not already in it
|
## add to queue only if not already in it
|
||||||
if [ $(grep -E ${file}$ ${TMPQUEUEFILE} | wc -l) -lt 1 ]; then
|
if [ $(grep -E ${file}$ ${tmpqueuefile} | wc -l) -lt 1 ]; then
|
||||||
#echo "Add notify to queue: file '$file' in directory '$dir' for '$action'" | tee -a ${TMPQUEUEFILE}
|
#echo "Add notify to queue: file '$file' in directory '$dir' for '$action'" | tee -a ${tmpqueuefile}
|
||||||
echo -e "[Masync log - ENQUEUE file] $dir$file"
|
echo -e "[Masync log - ENQUEUE file] $dir$file"
|
||||||
# AVOID TO USE TEE: SIMPLY append
|
# AVOID TO USE TEE: SIMPLY append
|
||||||
#echo ${dir}${file} | tee -a ${TMPQUEUEFILE}
|
#echo ${dir}${file} | tee -a ${tmpqueuefile}
|
||||||
echo ${dir}${file} >> ${TMPQUEUEFILE}
|
echo ${dir}${file} >> ${tmpqueuefile}
|
||||||
fi
|
fi
|
||||||
flock -u $lock_queue
|
flock -u $lock_queue
|
||||||
fi
|
fi
|
||||||
|
|
Loading…
Reference in New Issue