diff --git a/src/iface/__pycache__/snap.cpython-39.pyc b/src/iface/__pycache__/snap.cpython-39.pyc index 7669ffd..4a458a5 100644 Binary files a/src/iface/__pycache__/snap.cpython-39.pyc and b/src/iface/__pycache__/snap.cpython-39.pyc differ diff --git a/src/iface/snap.py b/src/iface/snap.py index 3024323..4c3c77e 100644 --- a/src/iface/snap.py +++ b/src/iface/snap.py @@ -1,3 +1,5 @@ +import os + from lib.snapshot import dump as _dump from lib.snapshot.generate import local as _genlocal @@ -21,7 +23,7 @@ def decode_snap(): return _dump.decode_snapshot(path=snap_path, dump_file_name = filename) -def diff_snap(last_tree, current_tree, path='./', bres={}): +def recursive_diff_snap(last_tree, current_tree, path='./', bres={}): ''' snapshot is a tree represented by dictionary {k, val} when k can be folder or file name and val can be a dictionary or an hash @@ -40,7 +42,7 @@ def diff_snap(last_tree, current_tree, path='./', bres={}): print(f'file {path}{key_added} subtree added in current snapshot') item = { 'name' : key_added, - # 'path' : '%s%s' % (path, name_last), + 'path' : '%s' % (path,), 'type' : 'dir' if isinstance(current_tree[key_added], dict) else 'file', 'hash' : current_tree[key_added], } @@ -96,16 +98,40 @@ def diff_snap(last_tree, current_tree, path='./', bres={}): print(f'file {path}{name_last} subtree changed in current snapshot') - diffsnap(hsh_last, hsh_current, path='%s%s/' % (path,name_last), bres = { + recursive_diff_snap(hsh_last, hsh_current, path='%s%s/' % (path,name_last), bres = { 'removed' : removed, 'added' : added, 'changed' : changed, }) +def diff_snap(last_tree, current_tree, path='./'): + res = {} + if not path.endswith(os.sep): path = path + os.sep + recursive_diff_snap(last_tree, current_tree, path=path, bres=res) + # compute moved could save bandwidth + res['moved'] = [] + for n, r in enumerate(res['removed']): + found, path = find_subtree(current_tree, { + r['name'] : r['hash'], + }) + if not found: continue + res['moved'].append({ + 'name' : r['name'], + 'old_path' : r['path'], + 'new_path' : path, + 'hash' : r['hash'], + }) + + del res['removed'][n] + + return res + def find_subtree(snapshot, subtree, path='./'): ''' @param subtree dict - find the exact subtree in snapshot + find the exact subtree in snapshot, subtree can be + - a node representing a file + - a node representing a folder ''' subtree_key = list(subtree.keys())[0] subtree_val = list(subtree.values())[0] @@ -114,7 +140,7 @@ def find_subtree(snapshot, subtree, path='./'): if snap_key != subtree_key: if isinstance(snap_val, dict): path = '%s%s/' % (path, snap_key) - return findsubtree(snap_val, subtree, path) + return find_subtree(snap_val, subtree, path) else: if snap_val == subtree_val: return (True, path) diff --git a/src/lib/diff/__pycache__/fdiff.cpython-39.pyc b/src/lib/diff/__pycache__/fdiff.cpython-39.pyc new file mode 100644 index 0000000..9df97cb Binary files /dev/null and b/src/lib/diff/__pycache__/fdiff.cpython-39.pyc differ diff --git a/src/lib/diff/__pycache__/mdiff.cpython-39.pyc b/src/lib/diff/__pycache__/mdiff.cpython-39.pyc index be2fabb..096fb74 100644 Binary files a/src/lib/diff/__pycache__/mdiff.cpython-39.pyc and b/src/lib/diff/__pycache__/mdiff.cpython-39.pyc differ diff --git a/src/lib/diff/fdiff.py b/src/lib/diff/fdiff.py new file mode 100644 index 0000000..647d2ce --- /dev/null +++ b/src/lib/diff/fdiff.py @@ -0,0 +1,118 @@ +from pathlib import Path +import difflib + +def compute_diff(filea_path, fileb_path): + with open(filea_path, 'r') as fa: + alines = fa.readlines() + with open(fileb_path, 'r') as fb: + blines = fb.readlines() + + differ = difflib.Differ() + res_diff = list(differ.compare(alines, blines)) + return res_diff + +def format_diff(filea_path, fileb_path, remove_diff_letters_code=False): + ''' + "- " line unique to file1 + "+ " line unique to file2 + " " line common to both files + "? " line not present in either input file -> to skip it + ''' + filea_obj = Path(filea_path) + fileb_obj = Path(fileb_path) + + filea_name = filea_obj.name + fileb_name = fileb_obj.name + + filea_group_wrap_tmpl = '<<<<<<< '+ filea_name + '\n%s=======\n' + fileb_group_wrap_tmpl = '>>>>>>> ' + fileb_name + '\n%s=======\n' + + + lines_diff = compute_diff(filea_path, fileb_path) + list_blocks = [] + + ''' + list blocks contains object like this + { + 'block_source' : `filea_name|fileb_name|common` + 'block_lines' : `lines of that block` + } + ''' + + for line in lines_diff: + last_block_index = len(list_blocks) + + if line.startswith('- '): + ## line in filea + block_source = filea_name + wrap_tmpl = filea_group_wrap_tmpl + elif line.startswith('+ '): + ## line in fileb + block_source = fileb_name + wrap_tmpl = fileb_group_wrap_tmpl + elif line.startswith('? '): + continue + elif line.startswith(' '): + ## common line for both files + block_source = 'common' + wrap_tmpl = '' + + if remove_diff_letters_code: + line = line[2:] + + if not last_block_index: + # add a block + list_blocks.append({ + 'block_source' : block_source, + 'block_lines' : [line], + 'block_wrap_tmpl' : wrap_tmpl, + }) + continue + + last_block = list_blocks[last_block_index-1] + + if last_block['block_source'] != block_source: + list_blocks.append({ + 'block_source' : block_source, + 'block_lines' : [line], + 'block_wrap_tmpl' : wrap_tmpl, + }) + continue + + ## block already exists, append the line + last_block['block_lines'].append(line) + + return list_blocks + +def print_diff(filea_path, fileb_path, remove_diff_letters_code=False, outfile=''): + formatted = format_diff(filea_path, fileb_path, remove_diff_letters_code=remove_diff_letters_code) + + out_str = '' + + for n,f in enumerate(formatted): + is_last = True if n == len(formatted) -1 else False + + source = f['block_source'] + lines = f['block_lines'] + + wrap_tmpl = f['block_wrap_tmpl'] + if wrap_tmpl: + if is_last: + ## strip the last \n in template + tmpl = wrap_tmpl[:-1] + else: + tmpl = wrap_tmpl + + out_str += tmpl % (''.join(lines),) + continue + + out_str += ''.join(lines) + + print(out_str) + if not outfile: return + + with open(outfile, 'w') as o: + o.write(out_str) + return + + diff --git a/src/lib/snapshot/generate/__pycache__/remote.cpython-39.pyc b/src/lib/snapshot/generate/__pycache__/remote.cpython-39.pyc index a3ca73e..3da2820 100644 Binary files a/src/lib/snapshot/generate/__pycache__/remote.cpython-39.pyc and b/src/lib/snapshot/generate/__pycache__/remote.cpython-39.pyc differ diff --git a/src/lib/snapshot/generate/remote.py b/src/lib/snapshot/generate/remote.py index e5b2db2..f0a212c 100644 --- a/src/lib/snapshot/generate/remote.py +++ b/src/lib/snapshot/generate/remote.py @@ -120,18 +120,18 @@ async def generate_tree_hash_oversftp_async(root_path :str): await asyncio.gather(*tasks) return rtreemap -def test_sync_rtree(): +def test_sync_rtree(path='/home/notanamber/notes_dev/'): start = time.time() print('Start task') - rtree = generate_tree_hash_oversftp('/home/notanamber/notes_dev/') + rtree = generate_tree_hash_oversftp(path) end = time.time() print('task done in %.2f' % (end - start)) return rtree -def test_async_rtree(): +def test_async_rtree(path='/home/notanamber/notes_dev/'): start = time.time() print('Start task') - rtree = asyncio.run(generate_tree_hash_oversftp_async('/home/notanamber/notes_dev/')) + rtree = asyncio.run(generate_tree_hash_oversftp_async(path)) end = time.time() print('task done in %.2f' % (end - start)) return rtree diff --git a/src/testing/__pycache__/diff_snap.cpython-39.pyc b/src/testing/__pycache__/diff_snap.cpython-39.pyc new file mode 100644 index 0000000..41d3e43 Binary files /dev/null and b/src/testing/__pycache__/diff_snap.cpython-39.pyc differ diff --git a/src/testing/diff_snap.py b/src/testing/diff_snap.py new file mode 100644 index 0000000..36790d8 --- /dev/null +++ b/src/testing/diff_snap.py @@ -0,0 +1,90 @@ +from iface import snap + + +def test_add_file(): + path = '/home/luca/test_diff' + last_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'source.txt': 'd41d8cd98f00b204e9800998ecf8427e', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1'}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'} + current_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'source.txt': 'd41d8cd98f00b204e9800998ecf8427e', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1', 'zoncazonca.txt' : 'b1129bae3d2522579947416bd421b77c'}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'} + + res = snap.diff_snap(last_tree, current_tree, path = path) + return res + +def test_add_folder(): + path = '/home/luca/test_diff' + last_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'source.txt': 'd41d8cd98f00b204e9800998ecf8427e', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1'}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'} + current_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'source.txt': 'd41d8cd98f00b204e9800998ecf8427e', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1', 'zoncazonca' : {'1' : 'b1129bae3d2522579947416bd421b77c', '2' : 'f2de3bf4027e0b76ae78e10b03ab0c82'}}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'} + + res = snap.diff_snap(last_tree, current_tree, path = path) + return res + +def test_remove_file(): + path = '/home/luca/test_diff' + current_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'source.txt': 'd41d8cd98f00b204e9800998ecf8427e', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1'}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'} + last_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'source.txt': 'd41d8cd98f00b204e9800998ecf8427e', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1', 'zoncazonca.txt' : 'b1129bae3d2522579947416bd421b77c'}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'} + + res = snap.diff_snap(last_tree, current_tree, path = path) + return res + +def test_remove_folder(): + path = '/home/luca/test_diff' + current_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'source.txt': 'd41d8cd98f00b204e9800998ecf8427e', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1'}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'} + last_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'source.txt': 'd41d8cd98f00b204e9800998ecf8427e', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1', 'zoncazonca' : {'1' : 'b1129bae3d2522579947416bd421b77c', '2' : 'f2de3bf4027e0b76ae78e10b03ab0c82'}}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'} + + res = snap.diff_snap(last_tree, current_tree, path = path) + return res + +def test_move_file(): + path = '/home/luca/test_diff' + current_tree = {'source.txt': 'd41d8cd98f00b204e9800998ecf8427e','file1.txt': '0a4b9e726e7f6efad41b5015ba08240d','diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1', 'zoncazonca' : {'1' : 'b1129bae3d2522579947416bd421b77c', '2' : 'f2de3bf4027e0b76ae78e10b03ab0c82'}}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'} + last_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1', 'zoncazonca' : {'1' : 'b1129bae3d2522579947416bd421b77c', '2' : 'f2de3bf4027e0b76ae78e10b03ab0c82','source.txt': 'd41d8cd98f00b204e9800998ecf8427e'}}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'} + + res = snap.diff_snap(last_tree, current_tree, path = path) + return res + +def test_move_folder(): + path = '/home/luca/test_diff' + current_tree = {'zoncazonca' : {'1' : 'b1129bae3d2522579947416bd421b77c', '2' : 'f2de3bf4027e0b76ae78e10b03ab0c82','source.txt': 'd41d8cd98f00b204e9800998ecf8427e'}, 'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1'}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'} + last_tree = {'file1.txt': '0a4b9e726e7f6efad41b5015ba08240d', 'diff.txt': '2ea73e06145254635c051de3232ce911', 'diff_cmd': 'f1da3af4027e0b76ae78e10b03ab0c82', 'folder': {'file1.txt': '163af31276c447901a7769a9645df394', 'folder1': {'zonca.txt': 'f89bd9df06590b8c12585e3dce5179c1', 'zoncazonca' : {'1' : 'b1129bae3d2522579947416bd421b77c', '2' : 'f2de3bf4027e0b76ae78e10b03ab0c82','source.txt': 'd41d8cd98f00b204e9800998ecf8427e'}}}, 'file2.txt': 'b1129bae3d2522579947416bd420a73b'} + + res = snap.diff_snap(last_tree, current_tree, path = path) + return res + + +def test_some_transformation(): + ''' + changed zonca.txt and moved foder1 into a new created folder ff + ''' + path = '/home/luca/test_diff/' + last_tree = { + "file1.txt": "0a4b9e726e7f6efad41b5015ba08240d", + "source.txt": "d41d8cd98f00b204e9800998ecf8427e", + "diff.txt": "2ea73e06145254635c051de3232ce911", + "diff_cmd": "f1da3af4027e0b76ae78e10b03ab0c82", + "folder": { + "file1.txt": "163af31276c447901a7769a9645df394", + "folder1": { + "zonca.txt": "f89bd9df06590b8c12585e3dce5179c1" + } + }, + "file2.txt": "b1129bae3d2522579947416bd420a73b" + } + + current_tree = { + "file1.txt": "0a4b9e726e7f6efad41b5015ba08240d", + "source.txt": "d41d8cd98f00b204e9800998ecf8427e", + "diff.txt": "2ea73e06145254635c051de3232ce911", + "diff_cmd": "f1da3af4027e0b76ae78e10b03ab0c82", + "file2.txt": "b1129bae3d2522579947416bd420a73b", + "ff": { + "folder": { + "file1.txt": "163af31276c447901a7769a9645df394", + "folder1": { + "zonca.txt": "2c50128790b60a0c89319f9cc8b31eb0" + # "zonca.txt": "f89bd9df06590b8c12585e3dce5179c1" + } + } + } + } + + res = snap.diff_snap(last_tree, current_tree, path = path) + return res