masync log

.filetemplates.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# Copyright 2021 Luca Paris
+#This file is part of masync.
+
+#masync is free software: you can redistribute it and/or modify
+#it under the terms of the GNU General Public License as published by
+#the Free Software Foundation, either version 3 of the License, or
+#(at your option) any later version.
+
+#masync is distributed in the hope that it will be useful,
+#but WITHOUT ANY WARRANTY; without even the implied warranty of
+#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#GNU General Public License for more details.
+
+#You should have received a copy of the GNU General Public License
+#along with masync.  If not, see <http://www.gnu.org/licenses/>.
+
+###############################################
+# the file contains all sync managed by masync
+###############################################
+SYNCFILE=~/.syncdir.sync
+###############################################
+# the file contains the snapshot. A snapshot is the list of files 
+# locally synchronized when the sync was stopped
+###############################################
+SNAPSHOTFILE=~/.snapshot_{{hash}}
+TMPQUEUEFILE=~/.syncdir_{{hash}}.queue
+TMPQUEUEDELETES=~/.syncdir_{{hash}}.deletes
+###############################################
+# the file when the loop logs the activity
+###############################################
+SYNCLOOPFILE=~/syncloop_{{hash}}.nohup
+###############################################
+# raw rsync log (used by .synccmd.sh)
+###############################################
+LOGFILERSYNC=~/syncdir_{{hash}}.log
+
+################################
+#  $1 -> instr, the string template to format, we expect it contains placeholders {{placeholder_name}}
+#  $[2..n] -> placeholder in the form placeholder_name=value
+################################
+
+format() {
+  instr=$1
+  shift
+  for arg in "$@"; do
+    IFS== read -r key val <<< $arg
+    instr=$(echo $instr | sed -e "s/{{$key}}/$val/g")
+  done
+  echo $instr
+}

.synccmd.sh

@@ -14,6 +14,8 @@
 
 #You should have received a copy of the GNU General Public License
 #along with masync.  If not, see <http://www.gnu.org/licenses/>.
+source ~/bin/.filetemplates.sh
+source ~/bin/.colordef.sh
 
 # $1 -> hash sync
 # $2 -> LOCAL PATH 
@@ -22,14 +24,21 @@ sync() {
   src=$2
   dest=$3
   otheropts=$4
-  LOGFILE=~/syncdir_$1.log
-  if [ ! -f $LOGFILE ]; then 
-    touch $LOGFILE
+  logfile=$(format ${LOGFILERSYNC} hash=$1)
+  syncloopfile=$(format ${SYNCLOOPFILE} hash=$1)
+  if [ ! -f $logfile ]; then 
+    touch $logfile
   fi
   # PUSH TO REMOTE - and PULL FROM REMOTE swapping `src` with `dest`
   #echo "executing rsync -aPu --log-file=$LOGFILE -e ssh $otheropts $src $dest"
   # NOTE REMOTEDIR ALREADY EXISTS IN DESTINATION
-  rsync -aPu --log-file=$LOGFILE -e ssh $otheropts $src $dest 1>/dev/null 2>&1
+  # in this form rsync receive only data from source 1>$syncloopfile 2>&1
+  rsync -i -aPu --progress --out-format="%i ${GREEN}%n%L${ENDCOLOR} %''b" --log-file=$logfile -e ssh $otheropts $src $dest | \
+    grep -E "(<|>|deleting)" | \
+    sed -E  "s~<(\w|\W){10}~\\${PURPLE}\[SEND\]\\${ENDCOLOR} to remote ~g" | \
+    sed -E  "s~>(\w|\W){10}~\\${PURPLE}\[RECEIVE\]\\${ENDCOLOR} from remote ~g" | \
+    sed -E  "s~\*deleting(\s)*~\\${PURPLE}\[DELETE\]\\${ENDCOLOR} in local ~g" | \
+    xargs -r -0 printf 1>>$syncloopfile 2>&1
   # last command execution exit code $?"
   rsync_result=$?
   if [[ ${rsync_result} -ne 0 ]]; then

masync

@@ -16,9 +16,8 @@
 #along with masync.  If not, see <http://www.gnu.org/licenses/>.
 
 source ~/bin/.synccmd.sh
+source ~/bin/.filetemplates.sh
 source ~/bin/.colordef.sh
-SYNCFILE=~/.syncdir.sync
-SNAPSHOTFILE=~/.snapshot_
 MYPID=$$
 STATUS_RUNNING=RUNNING
 STATUS_STOPPED=STOPPED
@@ -78,8 +77,8 @@ syncexists() {
 }
 
 feeddeletes() {
-  queuedeletes=~/.syncdir_$1.deletes
-  snapfile=${SNAPSHOTFILE}$1
+  queuedeletes=$(format ${TMPQUEUEDELETES} hash=$1)
+  snapfile=$(format ${SNAPSHOTFILE} hash=$1)
   grep $1 ${SYNCFILE} | while read hash pid when status localpath remotepath; do
     remoterelativepath=$(echo "$remotepath" | cut -d : -f 2)
     remotehost=$(echo "$remotepath" | cut -d : -f 1 | cut -d @ -f 2)
@@ -91,13 +90,31 @@ feeddeletes() {
       if [[ -e "${path}" ]]; then 
         tocheckremotepath=$(echo "${path}" | sed -e "s~${localpath}~${remoterelativepath}~g")
         if ! grep -Fxq "${tocheckremotepath}" ${remotefiles}; then 
-          echo -e "${PURPLE}[PURGE]${ENDCOLOR} ${path} in snapshot no longer exists in remote"; 
-          rm -rf "${path}"
+          # echo -e "${PURPLE}[PURGE]${ENDCOLOR} ${path} in snapshot no longer exists in remote"
+          echo -e -n "Do you want to remove ${RED}${path}${ENDCOLOR} from local sync? [y to remove,  return to skip it, N no to all] " 
+          read -u 1 ui
+          if [[ -z "${ui}" ]]; then
+            if [ ${ui} = 'y' ]; then
+              rm -rf "${path}"
+            fi
+            if [ ${ui} = 'N' ]; then
+              break
+            fi
+          fi
         fi
       else 
-        echo -e "${PURPLE}[PURGE]${ENDCOLOR} ${path} in snapshot no longer exists in local, delete in remote"; 
-        ## if controlled file doesn't exist in local delete it on remote
-        echo "${path}" | sed -e "s~${localpath}~${remoterelativepath}~g" | tee -a ${queuedeletes}
+        # echo -e "${PURPLE}[PURGE]${ENDCOLOR} ${path} in snapshot no longer exists in local, delete in remote"
+        echo -e -n "Do you want to remove ${RED}${path}${ENDCOLOR} from remote origin? [y to remove, return to skip it, N no to all] " 
+        read -u 1 ui
+        if [[ -z "${ui}" ]]; then
+          if [ ${ui} = 'y' ]; then
+            ## if controlled file doesn't exist in local delete it on remote
+            echo "${path}" | sed -e "s~${localpath}~${remoterelativepath}~g" | tee -a ${queuedeletes}
+          fi
+          if [ ${ui} = 'N' ]; then
+            break
+          fi
+        fi
       fi
     done < ${snapfile}
     rm -rf ${remotefiles}
@@ -140,23 +157,28 @@ initsyncpath() {
 startsyncpath() {
   echo -e "Start synching local folder ${GREEN}$1${ENDCOLOR} with remote ${GREEN}$2${ENDCOLOR}..."
   localpath_hash=$(echo "$1" | md5sum | cut -f1 --delimiter=" " -)
-  snapshotfile=${SNAPSHOTFILE}${localpath_hash}
-  tempqueuefile=~/.syncdir_${localpath_hash}.queue
+  # snapshotfile=${SNAPSHOTFILE}${localpath_hash}
+  snapshotfile=$(format ${SNAPSHOTFILE} hash=${localpath_hash})
+  #tempqueuefile=~/.syncdir_${localpath_hash}.queue
+  tempqueuefile=$(format ${TMPQUEUEFILE} hash=${localpath_hash})
+  #syncloopfile=$(format ${SYNCLOOPFILE} hash=${localpath_hash})
 
   if [ -e ${snapshotfile} ]; then 
-    echo "Founded snapshot file: ${snapshotfile} collect files to delete in origin"
+    echo "Founded snapshot file: ${snapshotfile} checking files to remove..."
     feeddeletes ${localpath_hash}
     echo "Remove snapshot file ${snapshotfile}"
     rm -f ${snapshotfile}
   fi
   ## ensure local computer sends eventually new files adding a fake line in the queue
+  #echo -e "${PURPLE}[DUMMY PUSH]${ENDCOLOR} to remote" >> ${syncloopfile}
   echo "Startsync DUMMY ACTION" >> ${tempqueuefile}
   echo -e "${GREEN}DONE${ENDCOLOR}"
   loopsyncpath $1
 }
 
 readlog() {
-  tail -f ~/syncloop_$1.nohup
+  syncloopfile=$(format ${SYNCLOOPFILE} hash=$1)
+  tail -f ${syncloopfile}
 }
 
 ##########################################
@@ -183,10 +205,12 @@ loopsyncpath() {
       echo 'Sync already running do nothing...'
       exit 0
     else
-      ## delete old log file
-      rm -f ~/syncdir_$hash.log
+      ## delete old rsync log file
+      rsynclogfile=$(format ${LOGFILERSYNC} hash=$hash)
+      rm -f ${rsynclogfile}
       ## starting loop on background and catch pid
-      nohup syncloop.sh $hash $localpath $remotepath 1>~/syncloop_$hash.nohup 2>&1 &
+      syncloopfile=$(format ${SYNCLOOPFILE} hash=$hash)
+      nohup syncloop.sh $hash $localpath $remotepath 1>$syncloopfile 2>&1 &
       mypid=$!
       when=$(date +%s)
       ## if line exists i must replace it with new pid
@@ -223,7 +247,8 @@ stoploopsyncpath() {
       when=$(date +%s)
       sed -i -e "$nline {s/$STATUS_RUNNING/$STATUS_STOPPED/; s/w[0-9]\+/w$when/}" "$SYNCFILE"
       #### SAVE THE SNAPSHOT
-      find ${localpath} > ${SNAPSHOTFILE}${hash}
+      snapshotfile=$(format ${SNAPSHOTFILE} hash=${hash})
+      find ${localpath} > ${snapshotfile}
     fi
   done
 }

syncloop.sh

@@ -21,22 +21,29 @@
 # @2   local path to sync
 # @3   remote path to sync
 #echo "$1, $2, $3"
+source ~/bin/.filetemplates.sh
 source ~/bin/.synccmd.sh
 source ~/bin/.colordef.sh
-TMPQUEUEFILE=~/.syncdir_$1.queue
-TMPQUEUEDELETES=~/.syncdir_$1.deletes
+tmpqueuefile=$(format ${TMPQUEUEFILE} hash=$1)
+tmpqueuedeletes=$(format ${TMPQUEUEDELETES} hash=$1)
+syncloopfile=$(format ${SYNCLOOPFILE} hash=$1)
+
+exec 3>&1 4>&2
+trap 'exec 2>&4 1>&3' 0 1 2 3
+exec 1>>$syncloopfile 2>&1
 
 LOCALPATH_HASH=$1
 LOCALPATH=$2
 REMOTEPATH=$3
 
+
 REMOTEHOST=$(echo "$REMOTEPATH" | cut -d : -f 1 | cut -d @ -f 2)
 REMOTERELATIVEPATH=$(echo "$REMOTEPATH" | cut -d : -f 2)
 
 #acquire lock_deletes file descritptor
-exec {lock_deletes}>>${TMPQUEUEDELETES}
+exec {lock_deletes}>>${tmpqueuedeletes}
 #acquire lock queue file descriptor
-exec {lock_queue}>>${TMPQUEUEFILE}
+exec {lock_queue}>>${tmpqueuefile}
 
 min() {
   if [ $1 -gt $2 ]; then 
@@ -58,42 +65,41 @@ synccycle() {
   thresholdpulllimit=600
   while :
   do
-    if [ -f "$TMPQUEUEFILE" ] || [ -f "$TMPQUEUEDELETES" ]; then 
+    if [ -f "$tmpqueuefile" ] || [ -f "$tmpqueuedeletes" ]; then 
       ### DELETES WHILE LOOP
-      if [ -f "$TMPQUEUEDELETES" ]; then 
-        ndeletes=$(wc -l < ${TMPQUEUEDELETES})
+      if [ -f "$tmpqueuedeletes" ]; then 
+        ndeletes=$(wc -l < ${tmpqueuedeletes})
         while [ $ndeletes -gt 0 ]; do
-          echo -n -e "[Masync log - PUSH FILES TO DELETE]  to remote: ${RED}$(head -n $ndeletes ${TMPQUEUEDELETES} | tr '\n', ' ')${ENDCOLOR}"
-          ssh $REMOTEHOST "rm -rf $(head -n $ndeletes ${TMPQUEUEDELETES} | tr '\n', ' ')"
-          echo -e " ${PURPLE}DONE${ENDCOLOR}"
+          echo -e "${PURPLE}[DELETE]${ENDCOLOR} at remote: ${RED}$(head -n $ndeletes ${tmpqueuedeletes} | tr '\n', ' ')${ENDCOLOR}"
+          ssh $REMOTEHOST "rm -rf $(head -n $ndeletes ${tmpqueuedeletes} | tr '\n', ' ')"
+          echo -e "${PURPLE}[DONE]${ENDCOLOR}"
           # remove the first ndeletes lines
           flock $lock_deletes
           tmpfile=$(mktemp)
-          tail -n +$(expr ${ndeletes} + 1) < ${TMPQUEUEDELETES} > ${tmpfile}
-          cat ${tmpfile} > ${TMPQUEUEDELETES}
+          tail -n +$(expr ${ndeletes} + 1) < ${tmpqueuedeletes} > ${tmpfile}
+          cat ${tmpfile} > ${tmpqueuedeletes}
           rm -f ${tmpfile}
           flock -u $lock_deletes
-          ndeletes=$(wc -l < ${TMPQUEUEDELETES})
+          ndeletes=$(wc -l < ${tmpqueuedeletes})
         done
       fi
 
       ### PUSH WHILE LOOP
-      if [ -f "$TMPQUEUEFILE" ]; then 
-        nqueue=$(wc -l < ${TMPQUEUEFILE})
+      if [ -f "$tmpqueuefile" ]; then 
+        nqueue=$(wc -l < ${tmpqueuefile})
         while [ $nqueue -gt 0 ]; do
-          #echo "${TMPQUEUEFILE} not empty consume ${nqueue} updates in queue"
-          echo -n -e "[Masync log - PUSH FILES] to remote: ${GREEN}$(head -n ${nqueue} ${TMPQUEUEFILE} | tr '\n', ' ')${ENDCOLOR}"
+          #echo "${tmpqueuefile} not empty consume ${nqueue} updates in queue"
           sync $LOCALPATH_HASH $LOCALPATH $REMOTEPATH 
-          echo -e " ${PURPLE}DONE${ENDCOLOR}"
+          echo -e "${PURPLE}[PUSH DONE]${ENDCOLOR}"
           # remove the first nqueue lines from queue
           flock $lock_queue
           tmpfile=$(mktemp)
-          tail -n +$(expr ${nqueue} + 1) < ${TMPQUEUEFILE} > ${tmpfile}
-          cat ${tmpfile} > ${TMPQUEUEFILE}
+          tail -n +$(expr ${nqueue} + 1) < ${tmpqueuefile} > ${tmpfile}
+          cat ${tmpfile} > ${tmpqueuefile}
           rm -f ${tmpfile}
           flock -u $lock_queue
           #lastsynctime="$(date -u +%s.%N)"
-          nqueue=$(wc -l < ${TMPQUEUEFILE})
+          nqueue=$(wc -l < ${tmpqueuefile})
         done
       fi
     fi
@@ -101,9 +107,9 @@ synccycle() {
     elapsed="$(bc <<<"$now-$lastsynctime")"
     ## if elapsed is greater than maxpulllimit sync from remote with delete option
     if [ 1 -eq $(echo "$elapsed>$maxpulllimit" | bc) ]; then
-      echo -n "[Masync log - PULL LIMIT reached] $maxpulllimit seconds reached: sync from remote with --delete option..."
+      echo -e "${PURPLE}[PULL LIMIT]${ENDCOLOR} $maxpulllimit seconds reached: sync from remote with --delete option..."
       sync $LOCALPATH_HASH $REMOTEPATH $LOCALPATH '--delete' 
-      echo -e " ${PURPLE}DONE${ENDCOLOR}"
+      echo -e "${PURPLE}[PULL LIMIT DONE]${ENDCOLOR}"
       lastsynctime="$(date -u +%s.%N)"
       ## increment the pull iteration or reset it
       if [ ${pulliteration} -eq  ${maxpulliteration} ]; then
@@ -123,19 +129,19 @@ synccycle &
 # exclude swp,swpx and 4913 files created by vim
 inotifywait -m -r -e create -e close_write -e move -e delete --exclude "\.swp|\.swx|4913|.txt~" $LOCALPATH | while read dir action file; do
     if [ $action = 'DELETE' ] || [ $action = 'DELETE,ISDIR' ]  || [ $action = 'MOVED_FROM' ] || [ $action = 'MOVED_FROM,ISDIR' ]; then
-      echo -e "[Masync log - ENQUEUE delete] $dir$file"
+      echo -e "syncloop - ENQUEUE delete $dir$file"
       flock $lock_deletes
-      echo $dir$file | sed -e "s~$LOCALPATH~$REMOTERELATIVEPATH~g" | tee -a ${TMPQUEUEDELETES} 1>/dev/null
+      echo $dir$file | sed -e "s~$LOCALPATH~$REMOTERELATIVEPATH~g" | tee -a ${tmpqueuedeletes} 1>/dev/null
       flock -u $lock_deletes
     else
       flock $lock_queue
       ## add to queue only if not already in it
-      if [ $(grep -E ${file}$ ${TMPQUEUEFILE} | wc -l) -lt 1 ]; then 
-        #echo "Add notify to queue: file '$file' in directory '$dir' for '$action'" | tee -a ${TMPQUEUEFILE}
-        echo -e "[Masync log - ENQUEUE file] $dir$file"
+      if [ $(grep -E ${file}$ ${tmpqueuefile} | wc -l) -lt 1 ]; then 
+        #echo "Add notify to queue: file '$file' in directory '$dir' for '$action'" | tee -a ${tmpqueuefile}
+        echo -e "syncloop - ENQUEUE file $dir$file"
         # AVOID TO USE TEE: SIMPLY append 
-        #echo ${dir}${file} | tee -a ${TMPQUEUEFILE}
-        echo ${dir}${file} >> ${TMPQUEUEFILE}
+        #echo ${dir}${file} | tee -a ${tmpqueuefile}
+        echo ${dir}${file} >> ${tmpqueuefile}
       fi
       flock -u $lock_queue
     fi