masy/syncloop.sh

#!/bin/bash
# Copyright 2021 Luca Paris
#This file is part of masync.

#masync is free software: you can redistribute it and/or modify
#it under the terms of the GNU General Public License as published by
#the Free Software Foundation, either version 3 of the License, or
#(at your option) any later version.

#masync is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#GNU General Public License for more details.

#You should have received a copy of the GNU General Public License
#along with masync.  If not, see <http://www.gnu.org/licenses/>.

## REQUIREMENT
# apt-get install inotify-tools
# @1   hash to retrieve sync
# @2   local path to sync
# @3   remote path to sync
#echo "$1, $2, $3"
source ~/bin/.filetemplates.sh
source ~/bin/.synccmd.sh
source ~/bin/.colordef.sh
tmpqueuefile=$(format ${TMPQUEUEFILE} hash=$1)
tmpqueuedeletes=$(format ${TMPQUEUEDELETES} hash=$1)
syncloopfile=$(format ${SYNCLOOPFILE} hash=$1)

exec 3>&1 4>&2
trap 'exec 2>&4 1>&3' 0 1 2 3
exec 1>>$syncloopfile 2>&1

LOCALPATH_HASH=$1
LOCALPATH=$2
REMOTEPATH=$3


REMOTEHOST=$(echo "$REMOTEPATH" | cut -d : -f 1 | cut -d @ -f 2)
REMOTERELATIVEPATH=$(echo "$REMOTEPATH" | cut -d : -f 2)

#acquire lock_deletes file descritptor
exec {lock_deletes}>>${tmpqueuedeletes}
#acquire lock queue file descriptor
exec {lock_queue}>>${tmpqueuefile}

min() {
  if [ $1 -gt $2 ]; then 
    echo $2
  else 
    echo $1
  fi
}

synccycle() {
  lastsynctime="$(date -u +%s.%N)"
  ### initial max waiting time for pull (seconds) used as unit
  ### first pulliteration zero cause immediate pulling from remote
  pulllimitunit=60
  pulliteration=0
  maxpulliteration=5
  let "maxpulllimit = $pulllimitunit * $pulliteration"
  ### max waiting time for pull (seconds) 10 min
  thresholdpulllimit=600
  while :
  do
    if [ -f "$tmpqueuefile" ] || [ -f "$tmpqueuedeletes" ]; then 
      ### DELETES WHILE LOOP
      if [ -f "$tmpqueuedeletes" ]; then 
        ndeletes=$(wc -l < ${tmpqueuedeletes})
        while [ $ndeletes -gt 0 ]; do
          echo -e "${PURPLE}[DELETE]${ENDCOLOR} at remote: ${RED}$(head -n $ndeletes ${tmpqueuedeletes} | tr '\n', ' ')${ENDCOLOR}"
          ssh $REMOTEHOST "rm -rf $(head -n $ndeletes ${tmpqueuedeletes} | tr '\n', ' ')"
          echo -e "${PURPLE}[DONE]${ENDCOLOR}"
          # remove the first ndeletes lines
          flock $lock_deletes
          tmpfile=$(mktemp)
          tail -n +$(expr ${ndeletes} + 1) < ${tmpqueuedeletes} > ${tmpfile}
          cat ${tmpfile} > ${tmpqueuedeletes}
          rm -f ${tmpfile}
          flock -u $lock_deletes
          ndeletes=$(wc -l < ${tmpqueuedeletes})
        done
      fi

      ### PUSH WHILE LOOP
      if [ -f "$tmpqueuefile" ]; then 
        nqueue=$(wc -l < ${tmpqueuefile})
        while [ $nqueue -gt 0 ]; do
          #echo "${tmpqueuefile} not empty consume ${nqueue} updates in queue"
          sync $LOCALPATH_HASH $LOCALPATH $REMOTEPATH 
          echo -e "${PURPLE}[PUSH DONE]${ENDCOLOR}"
          # remove the first nqueue lines from queue
          flock $lock_queue
          tmpfile=$(mktemp)
          tail -n +$(expr ${nqueue} + 1) < ${tmpqueuefile} > ${tmpfile}
          cat ${tmpfile} > ${tmpqueuefile}
          rm -f ${tmpfile}
          flock -u $lock_queue
          #lastsynctime="$(date -u +%s.%N)"
          nqueue=$(wc -l < ${tmpqueuefile})
        done
      fi
    fi
    now="$(date -u +%s.%N)"
    elapsed="$(bc <<<"$now-$lastsynctime")"
    ## if elapsed is greater than maxpulllimit sync from remote with delete option
    if [ 1 -eq $(echo "$elapsed>$maxpulllimit" | bc) ]; then
      echo -e "${PURPLE}[PULL LIMIT]${ENDCOLOR} $maxpulllimit seconds reached: sync from remote with --delete option..."
      sync $LOCALPATH_HASH $REMOTEPATH $LOCALPATH '--delete' 
      echo -e "${PURPLE}[PULL LIMIT DONE]${ENDCOLOR}"
      lastsynctime="$(date -u +%s.%N)"
      ## increment the pull iteration or reset it
      if [ ${pulliteration} -eq  ${maxpulliteration} ]; then
        pulliteration=0
      else
        let pulliteration++
      fi
      let "limit = $pulliteration * $pulliteration * $pulllimitunit"
      maxpulllimit=$(min $limit $thresholdpulllimit)
    fi
    sleep 10
  done
}

synccycle &

# exclude swp,swpx and 4913 files created by vim
inotifywait -m -r -e create -e close_write -e move -e delete --exclude "\.swp|\.swx|4913|.txt~" $LOCALPATH | while read dir action file; do
    if [ $action = 'DELETE' ] || [ $action = 'DELETE,ISDIR' ]  || [ $action = 'MOVED_FROM' ] || [ $action = 'MOVED_FROM,ISDIR' ]; then
      echo -e "syncloop - ENQUEUE delete $dir$file"
      flock $lock_deletes
      echo $dir$file | sed -e "s~$LOCALPATH~$REMOTERELATIVEPATH~g" | tee -a ${tmpqueuedeletes} 1>/dev/null
      flock -u $lock_deletes
    else
      flock $lock_queue
      ## add to queue only if not already in it
      if [ $(grep -E ${file}$ ${tmpqueuefile} | wc -l) -lt 1 ]; then 
        #echo "Add notify to queue: file '$file' in directory '$dir' for '$action'" | tee -a ${tmpqueuefile}
        echo -e "syncloop - ENQUEUE file $dir$file"
        # AVOID TO USE TEE: SIMPLY append 
        #echo ${dir}${file} | tee -a ${tmpqueuefile}
        echo ${dir}${file} >> ${tmpqueuefile}
      fi
      flock -u $lock_queue
    fi
  done