testing and other

src/iface/snap.py

@@ -1,3 +1,5 @@
+import os 
+
 from lib.snapshot import dump as _dump
 from lib.snapshot.generate import local as _genlocal
 
@@ -21,7 +23,7 @@ def decode_snap():
 
   return _dump.decode_snapshot(path=snap_path, dump_file_name = filename)
 
-def diff_snap(last_tree, current_tree, path='./', bres={}):
+def recursive_diff_snap(last_tree, current_tree, path='./', bres={}):
   '''
     snapshot is a tree represented by dictionary {k, val} when k can be folder or file name
     and val can be a dictionary or an hash
@@ -40,7 +42,7 @@ def diff_snap(last_tree, current_tree, path='./', bres={}):
       print(f'file {path}{key_added} subtree added in current snapshot')
       item = {
         'name' : key_added,
-        # 'path' : '%s%s' % (path, name_last),
+        'path' : '%s' % (path,),
         'type' : 'dir' if isinstance(current_tree[key_added], dict) else 'file',
         'hash' : current_tree[key_added],
       }
@@ -96,16 +98,40 @@ def diff_snap(last_tree, current_tree, path='./', bres={}):
 
     print(f'file {path}{name_last} subtree changed in current snapshot')
 
-    diffsnap(hsh_last, hsh_current, path='%s%s/' % (path,name_last), bres = {
+    recursive_diff_snap(hsh_last, hsh_current, path='%s%s/' % (path,name_last), bres = {
       'removed' : removed,
       'added' :  added,
       'changed' : changed,
       })
 
+def diff_snap(last_tree, current_tree, path='./'):
+  res = {}
+  if not path.endswith(os.sep): path = path + os.sep
+  recursive_diff_snap(last_tree, current_tree, path=path, bres=res)
+  # compute moved could save bandwidth
+  res['moved'] = []
+  for n, r in enumerate(res['removed']):
+    found, path = find_subtree(current_tree, {
+      r['name'] : r['hash'],
+      })
+    if not found: continue
+    res['moved'].append({
+      'name' : r['name'], 
+      'old_path' : r['path'],
+      'new_path' : path,
+      'hash' : r['hash'],
+      })
+
+    del res['removed'][n]
+  
+  return res
+
 def find_subtree(snapshot, subtree, path='./'):
   '''
     @param     subtree              dict
-    find the exact subtree in snapshot
+    find the exact subtree in snapshot, subtree can be 
+      - a node representing a file
+      - a node representing a folder
   '''
   subtree_key  = list(subtree.keys())[0]
   subtree_val = list(subtree.values())[0]
@@ -114,7 +140,7 @@ def find_subtree(snapshot, subtree, path='./'):
     if snap_key != subtree_key:
       if isinstance(snap_val, dict):
         path = '%s%s/' % (path, snap_key)
-        return findsubtree(snap_val, subtree, path)
+        return find_subtree(snap_val, subtree, path)
     else:
       if snap_val == subtree_val:
         return (True, path)

src/lib/diff/fdiff.py

@@ -0,0 +1,118 @@
+from pathlib import Path
+import difflib
+
+def compute_diff(filea_path, fileb_path):
+  with open(filea_path, 'r') as fa:
+    alines = fa.readlines()
+  with open(fileb_path, 'r') as fb:
+    blines = fb.readlines()
+
+  differ = difflib.Differ()
+  res_diff = list(differ.compare(alines, blines))
+  return res_diff
+
+def format_diff(filea_path, fileb_path, remove_diff_letters_code=False):
+  '''
+    "- "    line unique to file1
+    "+ "    line unique to file2
+    "  "    line common to both files
+    "? "    line not present in either input file -> to skip it
+  '''
+  filea_obj = Path(filea_path)
+  fileb_obj = Path(fileb_path)
+
+  filea_name = filea_obj.name
+  fileb_name = fileb_obj.name
+
+  filea_group_wrap_tmpl = '<<<<<<< '+ filea_name + '\n%s=======\n'
+  fileb_group_wrap_tmpl = '>>>>>>> ' + fileb_name + '\n%s=======\n'
+
+
+  lines_diff = compute_diff(filea_path, fileb_path)
+  list_blocks = []
+
+  '''
+    list blocks contains object like this
+    {
+    'block_source' : `filea_name|fileb_name|common`
+    'block_lines' : `lines of that block`
+    }
+  '''
+
+  for line in lines_diff:
+    last_block_index = len(list_blocks)
+
+    if line.startswith('- '):
+      ## line in filea
+      block_source = filea_name
+      wrap_tmpl = filea_group_wrap_tmpl
+    elif line.startswith('+ '):
+      ## line in fileb
+      block_source = fileb_name
+      wrap_tmpl = fileb_group_wrap_tmpl
+    elif line.startswith('? '):
+      continue
+    elif line.startswith('  '):
+      ## common line for both files
+      block_source = 'common'
+      wrap_tmpl = ''
+
+    if remove_diff_letters_code:
+      line = line[2:]
+
+    if not last_block_index:
+      # add a block
+      list_blocks.append({
+        'block_source' : block_source,
+        'block_lines' : [line],
+        'block_wrap_tmpl' : wrap_tmpl,
+      })
+      continue
+
+    last_block = list_blocks[last_block_index-1]
+
+    if last_block['block_source'] != block_source:
+      list_blocks.append({
+        'block_source' : block_source,
+        'block_lines' : [line],
+        'block_wrap_tmpl' : wrap_tmpl,
+      })
+      continue
+
+    ## block already exists, append the line
+    last_block['block_lines'].append(line)
+
+  return list_blocks
+
+def print_diff(filea_path, fileb_path, remove_diff_letters_code=False, outfile=''):
+  formatted = format_diff(filea_path, fileb_path, remove_diff_letters_code=remove_diff_letters_code)
+
+  out_str = ''
+
+  for n,f in enumerate(formatted):
+    is_last  = True if n == len(formatted) -1 else False
+
+    source = f['block_source']
+    lines = f['block_lines']
+
+    wrap_tmpl = f['block_wrap_tmpl']
+    if wrap_tmpl:
+      if is_last: 
+        ## strip the last \n in template
+        tmpl = wrap_tmpl[:-1]
+      else:
+        tmpl = wrap_tmpl
+
+      out_str += tmpl % (''.join(lines),)
+      continue
+
+    out_str += ''.join(lines)
+
+  print(out_str)
+  if not outfile: return
+
+  with open(outfile, 'w') as o:
+    o.write(out_str)
+  return 
+
+

src/lib/snapshot/generate/remote.py

@@ -120,18 +120,18 @@ async def generate_tree_hash_oversftp_async(root_path :str):
   await asyncio.gather(*tasks)
   return rtreemap
 
-def test_sync_rtree():
+def test_sync_rtree(path='/home/notanamber/notes_dev/'):
   start = time.time()
   print('Start task')
-  rtree = generate_tree_hash_oversftp('/home/notanamber/notes_dev/')
+  rtree = generate_tree_hash_oversftp(path)
   end = time.time()
   print('task done in %.2f' % (end - start))
   return rtree
 
-def test_async_rtree():
+def test_async_rtree(path='/home/notanamber/notes_dev/'):
   start = time.time()
   print('Start task')
-  rtree = asyncio.run(generate_tree_hash_oversftp_async('/home/notanamber/notes_dev/')) 
+  rtree = asyncio.run(generate_tree_hash_oversftp_async(path)) 
   end = time.time()
   print('task done in %.2f' % (end - start))
   return rtree

src/testing/diff_snap.py

@@ -0,0 +1,90 @@
+from iface import snap
+
+
+def test_add_file():
+  path = '/home/luca/test_diff'
+  last_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'source.txt': 'd41d8cd98f00b204e9800998ecf8427e', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1'}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'}
+  current_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'source.txt': 'd41d8cd98f00b204e9800998ecf8427e', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1', 'zoncazonca.txt' : 'b1129bae3d2522579947416bd421b77c'}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'}
+
+  res = snap.diff_snap(last_tree, current_tree, path = path)
+  return res
+
+def test_add_folder():
+  path = '/home/luca/test_diff'
+  last_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'source.txt': 'd41d8cd98f00b204e9800998ecf8427e', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1'}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'}
+  current_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'source.txt': 'd41d8cd98f00b204e9800998ecf8427e', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1', 'zoncazonca' : {'1' : 'b1129bae3d2522579947416bd421b77c', '2' : 'f2de3bf4027e0b76ae78e10b03ab0c82'}}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'}
+
+  res = snap.diff_snap(last_tree, current_tree, path = path)
+  return res
+
+def test_remove_file():
+  path = '/home/luca/test_diff'
+  current_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'source.txt': 'd41d8cd98f00b204e9800998ecf8427e', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1'}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'}
+  last_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'source.txt': 'd41d8cd98f00b204e9800998ecf8427e', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1', 'zoncazonca.txt' : 'b1129bae3d2522579947416bd421b77c'}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'}
+
+  res = snap.diff_snap(last_tree, current_tree, path = path)
+  return res
+
+def test_remove_folder():
+  path = '/home/luca/test_diff'
+  current_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'source.txt': 'd41d8cd98f00b204e9800998ecf8427e', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1'}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'}
+  last_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'source.txt': 'd41d8cd98f00b204e9800998ecf8427e', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1', 'zoncazonca' : {'1' : 'b1129bae3d2522579947416bd421b77c', '2' : 'f2de3bf4027e0b76ae78e10b03ab0c82'}}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'}
+
+  res = snap.diff_snap(last_tree, current_tree, path = path)
+  return res
+
+def test_move_file():
+  path = '/home/luca/test_diff'
+  current_tree = {'source.txt': 'd41d8cd98f00b204e9800998ecf8427e','file1.txt': '0a4b9e726e7f6efad41b5015ba08240d','diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1', 'zoncazonca' : {'1' : 'b1129bae3d2522579947416bd421b77c', '2' : 'f2de3bf4027e0b76ae78e10b03ab0c82'}}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'}
+  last_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1', 'zoncazonca' : {'1' : 'b1129bae3d2522579947416bd421b77c', '2' : 'f2de3bf4027e0b76ae78e10b03ab0c82','source.txt': 'd41d8cd98f00b204e9800998ecf8427e'}}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'}
+
+  res = snap.diff_snap(last_tree, current_tree, path = path)
+  return res
+
+def test_move_folder():
+  path = '/home/luca/test_diff'
+  current_tree = {'zoncazonca' : {'1' : 'b1129bae3d2522579947416bd421b77c', '2' : 'f2de3bf4027e0b76ae78e10b03ab0c82','source.txt': 'd41d8cd98f00b204e9800998ecf8427e'}, 'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1'}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'}
+  last_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1', 'zoncazonca' : {'1' : 'b1129bae3d2522579947416bd421b77c', '2' : 'f2de3bf4027e0b76ae78e10b03ab0c82','source.txt': 'd41d8cd98f00b204e9800998ecf8427e'}}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'}
+
+  res = snap.diff_snap(last_tree, current_tree, path = path)
+  return res
+
+
+def test_some_transformation():
+  '''
+    changed zonca.txt and moved foder1 into a new created folder ff
+  '''
+  path = '/home/luca/test_diff/'
+  last_tree = {
+    "file1.txt": "0a4b9e726e7f6efad41b5015ba08240d",
+    "source.txt": "d41d8cd98f00b204e9800998ecf8427e",
+    "diff.txt": "2ea73e06145254635c051de3232ce911",
+    "diff_cmd": "f1da3af4027e0b76ae78e10b03ab0c82",
+    "folder": {
+      "file1.txt": "163af31276c447901a7769a9645df394",
+      "folder1": {
+        "zonca.txt": "f89bd9df06590b8c12585e3dce5179c1"
+      }
+    },
+    "file2.txt": "b1129bae3d2522579947416bd420a73b"
+  }
+
+  current_tree = {
+    "file1.txt": "0a4b9e726e7f6efad41b5015ba08240d",
+    "source.txt": "d41d8cd98f00b204e9800998ecf8427e",
+    "diff.txt": "2ea73e06145254635c051de3232ce911",
+    "diff_cmd": "f1da3af4027e0b76ae78e10b03ab0c82",
+    "file2.txt": "b1129bae3d2522579947416bd420a73b",
+    "ff": {
+      "folder": {
+        "file1.txt": "163af31276c447901a7769a9645df394",
+        "folder1": {
+          "zonca.txt": "2c50128790b60a0c89319f9cc8b31eb0"
+          # "zonca.txt": "f89bd9df06590b8c12585e3dce5179c1"
+        }
+      }
+    }
+  }
+
+  res = snap.diff_snap(last_tree, current_tree, path = path)
+  return res