diff --git a/.gitignore b/.gitignore index d9568ca..35a3ed6 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ *swp +*.pyc +__pycache__ diff --git a/src/iface/__pycache__/snap.cpython-39.pyc b/src/iface/__pycache__/snap.cpython-39.pyc deleted file mode 100644 index 546396f..0000000 Binary files a/src/iface/__pycache__/snap.cpython-39.pyc and /dev/null differ diff --git a/src/iface/__pycache__/sync_manager.cpython-39.pyc b/src/iface/__pycache__/sync_manager.cpython-39.pyc deleted file mode 100644 index 0ca587c..0000000 Binary files a/src/iface/__pycache__/sync_manager.cpython-39.pyc and /dev/null differ diff --git a/src/iface/sync_manager.py b/src/iface/sync_manager.py index aca4c36..4fc06d5 100644 --- a/src/iface/sync_manager.py +++ b/src/iface/sync_manager.py @@ -133,6 +133,13 @@ class Manager(): snapshot = _dump.load_snapshot(snapshot_dump_path) return snapshot + def get_unmerged_local_path(self, node): + node_path = node['path'] + node_name = node['name'] + unmerged_local_file_path = '%s%s' % (node_path, node_name) + unmerged_file_path = unmerged_local_file_path.replace('.%s' % (os.sep), self.get_unmerged_path()) + return unmerged_file_path + def get_absolute_local_path(self, node): node_path = node['path'] node_name = node['name'] @@ -189,13 +196,31 @@ class Manager(): def get_remote_snap_diff(self): pass + def store_fdiff(self, node, only_print=False): + ''' + store in file diff path `.masy/diff` the tree with diffs + file is containing the diff between local and remote + ''' + node_local_path = self.get_absolute_local_path(node) + filea_block_tag = '%s %s (local)' % (node['name'], node['cur_hash']) + fileb_block_tag = '%s %s (remote)' % (node['name'], node['remote_hash']) + rfile_buf = node.pop('remote_file_buf') + node_diff_path = self.get_tree_diff_path(node) + try: + os.makedirs(node_diff_path) + except FileExistsError: + print(f'{node_diff_path} already exists skip creation') + + outfile = '' + if not only_print: + outfile = '%s%s' % (self.normalize_path(node_diff_path), node['name']) + fdiff.print_diff(node_local_path, rfile_buf, remove_diff_letters_code=False, outfile=outfile, filea_block_tag=filea_block_tag, fileb_block_tag=fileb_block_tag) + return node + def store_unmerged_diff(self, nodes): ''' node_path is the relative path ''' - # path = node['path'] - # name = node['name'] - unmerged_path = self.get_unmerged_path() # unmerged_path = Path(self.local_path) # if not local_path.exists(): @@ -209,18 +234,8 @@ class Manager(): todump = [] for node in nodes: - node_local_path = self.get_absolute_local_path(node) - filea_block_tag = '%s %s (local)' % (node['name'], node['cur_hash']) - fileb_block_tag = '%s %s (remote)' % (node['name'], node['remote_hash']) - rfile_buf = node.pop('remote_file') - node_diff_path = self.get_tree_diff_path(node) - try: - os.makedirs(node_diff_path) - except FileExistsError: - print(f'{node_diff_path} already exists skip creation') - outfile = '%s%s' % (self.normalize_path(node_diff_path), node['name']) - fdiff.print_diff(node_local_path, rfile_buf, remove_diff_letters_code=False, outfile=outfile, filea_block_tag=filea_block_tag, fileb_block_tag=fileb_block_tag) - todump.append(node) + n = self.store_fdiff(node) + todump.append(n) _dump.dump_snapshot(todump, path=unmerged_path, dump_file_name='.unmerged.json.gz') @@ -245,8 +260,6 @@ class Manager(): ## node is a folder return agent.check_rfolder_status(absolute_remote_path) - # if node_type == 'f': - def copy_node(self, node): pass # node_type = node['type'] @@ -258,6 +271,57 @@ class Manager(): # if node_type == 'd': # pass + def show_conflicts(self): + unmerged = self.load_unmerged_diff() + for n, unode in enumerate(unmerged): + i = n+1 + path = unode['path'] + name = unode['name'] + remote_hash = unode['remote_hash'] + last_type = unode['last_type'] + print(f'{i}- path: \'{path}\', name: \'{name}\', last_type: \'{last_type}\' remote_hash: \'{remote_hash}\'') + + def mark_conflicting_node_as_solved(self, node): + name = node['name'] + path = node['path'] + + unmerged = self.load_unmerged_diff() + + found = {} + for n, unode in enumerate(unmerged): + if unode['name'] == name and unode['path'] == path: + found = unode + break + else: + n = -1 + + if n == -1: + return 'No node conflicting' + + remote_hash = found['remote_hash'] + ## if you mark the node as conflict solved + ## you have incorporated the `remote_hash` + ## you must verify this is the actual hash in the server side + rstatus = self.get_rnode_status(node) + rfile_buf, rhash, rexists = map(rstatus.get, ['iobuffer', 'hash', 'exists']) + + if not (remote_hash == rhash): + print(f'remote hash changed you...generate a new conflict') + ## update the conflicting node by index + (cur_hash, cur_buf) = _genlocal.generate_file_hash(self.get_absolute_local_path(found)) + found['cur_hash'] = self.get_absolute_local_path(found) + found['last_hash'] = found['remote_hash'] + found['remote_hash'] = rhash + found['remote_file_buf'] = rfile_buf + self.store_unmerged_diff(unmerged) + ## store diff in diffs path + self.store_fdiff(found) + return + ## found the node in unmerged path + unmerged_local_path = self.get_unmerged_local_path(node) + absolute_remote_path = self.get_absolute_remote_path(node) + print(f'force local node: {unmerged_local_path} to remote path: {absolute_remote_path}') + def sync(self): @@ -281,51 +345,73 @@ class Manager(): for node in changes: node_name = node['name'] - node_path = node['path'] - node_current_type = node['current_type'] - node_last_type = node['last_type'] - node_current_hash = node['cur_hash'] - node_last_hash = node['last_hash'] + last_hash = node['last_hash'] + print(f'Checking local {node_name}') - if not (node_current_type == node_last_type): - print(f'node {node_name} change type in local tree') - if node_last_type == 'f': - # remote_file_path = '%s%s' % (node_path, node_name) - # remote_file_path = remote_file_path.replace('.%s' % (os.sep), self.remote_path) - remote_file_path = self.get_absolute_remote_path(node) - rfile_buf, rhash = agent.generate_file_hash_oversftp(remote_file_path, return_also_buffer=True) - if node_last_hash == rhash: - print(f'You can proceed to push {node_name} file it is not changed from the last version') - # local_file_path = '%s%s' % (node_path, node_name) - # local_file_path = local_file_path.replace('.%s' % (os.sep), self.local_path) - local_file_path = self.get_absolute_local_path(node) - agent.put(local_file_path, remote_file_path, lambda x,y: print(f'{local_file_path} copied correctly to remote')) - else: - print(f'{node_path} file it changed local hash: {node_last_hash}, remote hash {rhash}, you can\'t push it directly') - # self.store_unmerged_diff(node) - node['remote_hash'] = rhash - node['remote_file'] = rfile_buf - unmerged.append(node) + rstatus = self.get_rnode_status(node) + rfile_buf, rhash, rexists = map(rstatus.get, ['iobuffer', 'hash', 'exists']) + node['remote_hash'] = rhash + node['remote_file_buf'] = rfile_buf - if node_last_type == 'd': - # remote_path = '%s%s' % (node_path, node_name) - # remote_path = remote_file_path.replace('.%s' % (os.sep), self.remote_path) - remote_path = self.get_absolute_local_path(node) - rhash = agent.generate_tree_hash_oversftp(remote_path) - if node_last_hash == rhash: - print(f'You can proceed to push {node_name} folder it is not changed from the last version') - else: - print(f'{node_name} folder it changed from the last version, you can\'t push it directly') + if not rexists or last_hash != rhash: + print(f'Put node {node_name} in unmerged, node not exists in remote or is changed from the last local snapshot') + unmerged.append(node) + else: + print(f'You can proceed to push {node_name} file it is not changed from the last version') if unmerged: self.store_unmerged_diff(unmerged) - ## managing added - added = local_snap_diff.get('added') or [] - for node in added: - ## if node not exists in remote copy it recursively in remote - remote_file_path = self.get_absolute_remote_path(node) - agent.copy(node) + return - return local_snap_diff + + # for node in changes: + # node_name = node['name'] + # node_path = node['path'] + # node_current_type = node['current_type'] + # node_last_type = node['last_type'] + # node_current_hash = node['cur_hash'] + # node_last_hash = node['last_hash'] + # + # if not (node_current_type == node_last_type): + # print(f'node {node_name} change type in local tree') + # if node_last_type == 'f': + # # remote_file_path = '%s%s' % (node_path, node_name) + # # remote_file_path = remote_file_path.replace('.%s' % (os.sep), self.remote_path) + # remote_file_path = self.get_absolute_remote_path(node) + # rfile_buf, rhash = agent.generate_file_hash_oversftp(remote_file_path, return_also_buffer=True) + # if node_last_hash == rhash: + # print(f'You can proceed to push {node_name} file it is not changed from the last version') + # # local_file_path = '%s%s' % (node_path, node_name) + # # local_file_path = local_file_path.replace('.%s' % (os.sep), self.local_path) + # local_file_path = self.get_absolute_local_path(node) + # agent.put(local_file_path, remote_file_path, lambda x,y: print(f'{local_file_path} copied correctly to remote')) + # else: + # print(f'{node_path} file it changed local hash: {node_last_hash}, remote hash {rhash}, you can\'t push it directly') + # # self.store_unmerged_diff(node) + # node['remote_hash'] = rhash + # node['remote_file'] = rfile_buf + # unmerged.append(node) + # + # if node_last_type == 'd': + # # remote_path = '%s%s' % (node_path, node_name) + # # remote_path = remote_file_path.replace('.%s' % (os.sep), self.remote_path) + # remote_path = self.get_absolute_local_path(node) + # rhash = agent.generate_tree_hash_oversftp(remote_path) + # if node_last_hash == rhash: + # print(f'You can proceed to push {node_name} folder it is not changed from the last version') + # else: + # print(f'{node_name} folder it changed from the last version, you can\'t push it directly') + # + # if unmerged: + # self.store_unmerged_diff(unmerged) + # + # ## managing added + # added = local_snap_diff.get('added') or [] + # for node in added: + # ## if node not exists in remote copy it recursively in remote + # remote_file_path = self.get_absolute_remote_path(node) + # agent.copy(node) + # + # return local_snap_diff diff --git a/src/lib/diff/__pycache__/fdiff.cpython-39.pyc b/src/lib/diff/__pycache__/fdiff.cpython-39.pyc deleted file mode 100644 index ea0a279..0000000 Binary files a/src/lib/diff/__pycache__/fdiff.cpython-39.pyc and /dev/null differ diff --git a/src/lib/diff/__pycache__/mdiff.cpython-39.pyc b/src/lib/diff/__pycache__/mdiff.cpython-39.pyc deleted file mode 100644 index 096fb74..0000000 Binary files a/src/lib/diff/__pycache__/mdiff.cpython-39.pyc and /dev/null differ diff --git a/src/lib/repr/node.py b/src/lib/repr/node.py new file mode 100644 index 0000000..ee2504c --- /dev/null +++ b/src/lib/repr/node.py @@ -0,0 +1,23 @@ +from dataclasses import dataclass + +@dataclass +class Node: + name: str + rel_path: str + last_type: str + cur_type:str + last_hash: str + cur_hash: str + remote_hash: str = None + ''' + A node is a representation of a node within the tree + that we wish to keep synchronised with remote source + + @param name name of node + @param rel_path relative path of node respect of root of shared folder + @param last_type is the last type received from server + @param cur_type is the current type in local tree + @param last_hash is the last hash received from server (a directory can have or empty hash or a subtree) + @param cur_hash is the current hash in local tree + @param remote_hash is the remote hash (usually computed with a remote call) + ''' diff --git a/src/lib/sclient/__pycache__/agent.cpython-39.pyc b/src/lib/sclient/__pycache__/agent.cpython-39.pyc deleted file mode 100644 index 9ac3cef..0000000 Binary files a/src/lib/sclient/__pycache__/agent.cpython-39.pyc and /dev/null differ diff --git a/src/lib/sclient/__pycache__/base_agent.cpython-39.pyc b/src/lib/sclient/__pycache__/base_agent.cpython-39.pyc deleted file mode 100644 index 9454a88..0000000 Binary files a/src/lib/sclient/__pycache__/base_agent.cpython-39.pyc and /dev/null differ diff --git a/src/lib/snapshot/__pycache__/dump.cpython-39.pyc b/src/lib/snapshot/__pycache__/dump.cpython-39.pyc deleted file mode 100644 index 5b810f1..0000000 Binary files a/src/lib/snapshot/__pycache__/dump.cpython-39.pyc and /dev/null differ diff --git a/src/lib/snapshot/__pycache__/gen.cpython-39.pyc b/src/lib/snapshot/__pycache__/gen.cpython-39.pyc deleted file mode 100644 index 9a8cc92..0000000 Binary files a/src/lib/snapshot/__pycache__/gen.cpython-39.pyc and /dev/null differ diff --git a/src/lib/snapshot/__pycache__/snap.cpython-39.pyc b/src/lib/snapshot/__pycache__/snap.cpython-39.pyc deleted file mode 100644 index 26e7245..0000000 Binary files a/src/lib/snapshot/__pycache__/snap.cpython-39.pyc and /dev/null differ diff --git a/src/lib/snapshot/generate/__pycache__/local.cpython-39.pyc b/src/lib/snapshot/generate/__pycache__/local.cpython-39.pyc deleted file mode 100644 index ff61c07..0000000 Binary files a/src/lib/snapshot/generate/__pycache__/local.cpython-39.pyc and /dev/null differ diff --git a/src/lib/snapshot/generate/__pycache__/remote.cpython-39.pyc b/src/lib/snapshot/generate/__pycache__/remote.cpython-39.pyc deleted file mode 100644 index ef37646..0000000 Binary files a/src/lib/snapshot/generate/__pycache__/remote.cpython-39.pyc and /dev/null differ diff --git a/src/lib/snapshot/generate/local.py b/src/lib/snapshot/generate/local.py index 1d94504..1a2e710 100644 --- a/src/lib/snapshot/generate/local.py +++ b/src/lib/snapshot/generate/local.py @@ -1,15 +1,20 @@ import os +import io import hashlib import json import gzip -def generate_file_hash(file_path, hexdigest=True): +def generate_file_hash(file_path, hexdigest=True, return_also_buffer=False): with open(file_path, "rb") as f: buf = f.read() - if hexdigest: return hashlib.md5(buf).hexdigest() - return hashlib.md5(buf).digest() + if hexdigest: + if not return_also_buffer: return hashlib.md5(buf).hexdigest() + return (hashlib.md5(buf).hexdigest(), io.BytesIO(buf)) + + if not return_also_buffer: return hashlib.md5(buf).digest() + return (hashlib.md5(buf).digest(), io.BytesIO(buf)) def check_isdir(path: str): if not os.path.isdir(path): diff --git a/src/lib/snapshot/generate/remote.py b/src/lib/snapshot/generate/remote.py index 31b856f..fa0fe3b 100644 --- a/src/lib/snapshot/generate/remote.py +++ b/src/lib/snapshot/generate/remote.py @@ -67,17 +67,19 @@ class RHAgent(): dig = self.generate_tree_hash_oversftp(path) return { 'exists' : True, - # 'iobuffer' : iobuf, + 'iobuffer' : None, 'hash': dig, } except IOError as e: if e.errno == errno.ENOENT: return { 'exists' : False, - # 'iobuffer' : None, + 'iobuffer' : None, 'hash': None } raise Exception(f'Something went wrong and strange {path}') + def copy_node(self): + pass def generate_tree_hash_oversftp(self, root_path :str): diff --git a/src/testing/__pycache__/diff_snap.cpython-39.pyc b/src/testing/__pycache__/diff_snap.cpython-39.pyc deleted file mode 100644 index 41d3e43..0000000 Binary files a/src/testing/__pycache__/diff_snap.cpython-39.pyc and /dev/null differ diff --git a/src/testing/sync.py b/src/testing/sync.py index bd95556..6823a26 100644 --- a/src/testing/sync.py +++ b/src/testing/sync.py @@ -1,5 +1,9 @@ from iface import sync_manager +def get_test_manager(): + m = sync_manager.Manager(local_path='/home/luca/sharednotes_dev', remote_path='notanamber@myvps:/home/notanamber/notes_dev') + return m + def test_init_sync(): m = sync_manager.Manager(local_path='/home/luca/sharednotes_dev', remote_path='notanamber@myvps:/home/notanamber/notes_dev') m.init_sync() diff --git a/src/tools/__pycache__/parse_conf.cpython-39.pyc b/src/tools/__pycache__/parse_conf.cpython-39.pyc deleted file mode 100644 index 4bbe207..0000000 Binary files a/src/tools/__pycache__/parse_conf.cpython-39.pyc and /dev/null differ diff --git a/src/tools/__pycache__/pretty_print.cpython-39.pyc b/src/tools/__pycache__/pretty_print.cpython-39.pyc deleted file mode 100644 index 74095f9..0000000 Binary files a/src/tools/__pycache__/pretty_print.cpython-39.pyc and /dev/null differ diff --git a/src/tools/__pycache__/tree_repr.cpython-39.pyc b/src/tools/__pycache__/tree_repr.cpython-39.pyc deleted file mode 100644 index c57dea9..0000000 Binary files a/src/tools/__pycache__/tree_repr.cpython-39.pyc and /dev/null differ