diff --git a/src/iface/sync_manager.py b/src/iface/sync_manager.py index 3705e7d..e8cce5d 100644 --- a/src/iface/sync_manager.py +++ b/src/iface/sync_manager.py @@ -157,6 +157,11 @@ class Manager(): remote_file_path = remote_file_path.replace('.%s' % (os.sep), self.remote_path) return remote_file_path + def get_remote_mount_point(self, node): + node_path = node['rel_path'] + remote_mount_point = node_path.replace('.%s' % (os.sep), self.remote_path) + return remote_mount_point + def add_sync_to_list(self): home_dir = os.environ.get('HOME') home_dir = self.normalize_path(home_dir) @@ -257,7 +262,9 @@ class Manager(): ''' absolute_remote_path = self.get_absolute_remote_path(node) agent = self.get_agent() - node_type = node['last_type'] + ## if node not exist in the local tree there is not last_type + ## we use cur_type (is this correct?) + node_type = node.get('last_type') or node['cur_type'] if node_type == 'f': return agent.check_rfile_status(absolute_remote_path) ## node is a folder @@ -273,11 +280,12 @@ class Manager(): agent = self.get_agent() if node['cur_type'] == 'f': ## simply put - return agent.copy_file_to_remote(absolute_local_path, absolute_remote_path) + return agent.copy_file(absolute_local_path, absolute_remote_path) elif node['cur_type'] == 'd': #cur_hash contains the entire subtree to recreate on remote ## it is better to remove the root node if already exists on remote - agent.remove_dir(absolute_remote_path) + remote_mount_point = self.get_remote_mount_point(node) + return agent.copy_dir(absolute_local_path, remote_mount_point) def show_conflicts(self): @@ -333,7 +341,53 @@ class Manager(): print(f'force local node: {unmerged_local_path} to remote path: {absolute_remote_path}') self.copy_node_to_remote(node) + def sync_added(self, added=False): + if added is None: + local_snap_diff = self.get_local_snap_diff() + changes = local_snap_diff.get('added') or [] + for node in added: + node_name = node['name'] + last_hash = node.get('last_hash') + if last_hash is not None: + raise Exception('node added has a last hash!!') + print(f'Checking local {node_name}') + + rstatus = self.get_rnode_status(node) + rfile_buf, rhash, rexists = map(rstatus.get, ['iobuffer', 'hash', 'exists']) + if rexists: + raise Exception('This case must be managed!!! remote node appeared but in the local tree there was not') + + self.copy_node_to_remote(node) + + + def sync_changed(self, changes=None): + if changes is None: + local_snap_diff = self.get_local_snap_diff() + changes = local_snap_diff.get('changed') or [] + + unmerged = [] + + for node in changes: + node_name = node['name'] + last_hash = node['last_hash'] + print(f'Checking local {node_name}') + + rstatus = self.get_rnode_status(node) + rfile_buf, rhash, rexists = map(rstatus.get, ['iobuffer', 'hash', 'exists']) + node['remote_hash'] = rhash + node['remote_file_buf'] = rfile_buf + + if not rexists or last_hash != rhash: + print(f'Put node {node_name} in unmerged, node not exists in remote or is changed from the last local snapshot') + unmerged.append(node) + continue + + print(f'You can proceed to push {node_name} file it is not changed from the last version') + self.copy_node_to_remote(node) + + if unmerged: + self.store_unmerged_diff(unmerged) def sync(self): ''' @@ -351,78 +405,12 @@ class Manager(): # rhagent = self.init_rh_agent() # agent = self.get_agent() changes = local_snap_diff.get('changed') or [] + self.sync_changed(changes=changes) - unmerged = [] + added = local_snap_diff.get('added') or [] + self.sync_added(added=added) - for node in changes: - node_name = node['name'] - last_hash = node['last_hash'] - print(f'Checking local {node_name}') - - rstatus = self.get_rnode_status(node) - rfile_buf, rhash, rexists = map(rstatus.get, ['iobuffer', 'hash', 'exists']) - node['remote_hash'] = rhash - node['remote_file_buf'] = rfile_buf - - if not rexists or last_hash != rhash: - print(f'Put node {node_name} in unmerged, node not exists in remote or is changed from the last local snapshot') - unmerged.append(node) - else: - print(f'You can proceed to push {node_name} file it is not changed from the last version') - - if unmerged: - self.store_unmerged_diff(unmerged) return - - # for node in changes: - # node_name = node['name'] - # node_path = node['path'] - # node_current_type = node['current_type'] - # node_last_type = node['last_type'] - # node_current_hash = node['cur_hash'] - # node_last_hash = node['last_hash'] - # - # if not (node_current_type == node_last_type): - # print(f'node {node_name} change type in local tree') - # if node_last_type == 'f': - # # remote_file_path = '%s%s' % (node_path, node_name) - # # remote_file_path = remote_file_path.replace('.%s' % (os.sep), self.remote_path) - # remote_file_path = self.get_absolute_remote_path(node) - # rfile_buf, rhash = agent.generate_file_hash_oversftp(remote_file_path, return_also_buffer=True) - # if node_last_hash == rhash: - # print(f'You can proceed to push {node_name} file it is not changed from the last version') - # # local_file_path = '%s%s' % (node_path, node_name) - # # local_file_path = local_file_path.replace('.%s' % (os.sep), self.local_path) - # local_file_path = self.get_absolute_local_path(node) - # agent.put(local_file_path, remote_file_path, lambda x,y: print(f'{local_file_path} copied correctly to remote')) - # else: - # print(f'{node_path} file it changed local hash: {node_last_hash}, remote hash {rhash}, you can\'t push it directly') - # # self.store_unmerged_diff(node) - # node['remote_hash'] = rhash - # node['remote_file'] = rfile_buf - # unmerged.append(node) - # - # if node_last_type == 'd': - # # remote_path = '%s%s' % (node_path, node_name) - # # remote_path = remote_file_path.replace('.%s' % (os.sep), self.remote_path) - # remote_path = self.get_absolute_local_path(node) - # rhash = agent.generate_tree_hash_oversftp(remote_path) - # if node_last_hash == rhash: - # print(f'You can proceed to push {node_name} folder it is not changed from the last version') - # else: - # print(f'{node_name} folder it changed from the last version, you can\'t push it directly') - # - # if unmerged: - # self.store_unmerged_diff(unmerged) - # - # ## managing added - # added = local_snap_diff.get('added') or [] - # for node in added: - # ## if node not exists in remote copy it recursively in remote - # remote_file_path = self.get_absolute_remote_path(node) - # agent.copy(node) - # - # return local_snap_diff diff --git a/src/lib/diff/fdiff.py b/src/lib/diff/fdiff.py index d42800e..c952af8 100644 --- a/src/lib/diff/fdiff.py +++ b/src/lib/diff/fdiff.py @@ -11,6 +11,7 @@ def is_buf(f): return False def get_lines(file): + if not file: return [] if is_buf(file): return file.readlines() with open(file, 'r') as f: @@ -21,8 +22,16 @@ def compute_diff(filea, fileb): ''' file* can be a path or a buffer ''' - alines = get_lines(filea) - blines = get_lines(fileb) + try: + alines = get_lines(filea) + except: + print('alines error: file could be a binary') + alines = ['binary content ---\n'] + try: + blines = get_lines(fileb) + except UnicodeDecodeError as e: + print('blines error: file could be a binary') + blines = ['--- binary content\n'] differ = difflib.Differ() res_diff = list(differ.compare(alines, blines)) @@ -47,7 +56,6 @@ def format_diff(filea, fileb, remove_diff_letters_code=False, filea_block_tag='' "? " line not present in either input file -> to skip it ''' - filea_block_tag = get_file_block_tag(filea, filea_block_tag) fileb_block_tag = get_file_block_tag(fileb, fileb_block_tag) diff --git a/src/lib/repr/node.py b/src/lib/repr/node.py index ee2504c..a438522 100644 --- a/src/lib/repr/node.py +++ b/src/lib/repr/node.py @@ -18,6 +18,8 @@ class Node: @param last_type is the last type received from server @param cur_type is the current type in local tree @param last_hash is the last hash received from server (a directory can have or empty hash or a subtree) - @param cur_hash is the current hash in local tree + @param cur_hash is the current hash in local tree (in local the hash has changed from `last_hash` to `cur_hash`) @param remote_hash is the remote hash (usually computed with a remote call) + no conflict -> `remote_hash` == `last_hash` + conflict -> `remote_hash` != `last_hash` this means we have lost a version of node ''' diff --git a/src/lib/sclient/agent.py b/src/lib/sclient/agent.py index 1a11fb3..af4e4f8 100644 --- a/src/lib/sclient/agent.py +++ b/src/lib/sclient/agent.py @@ -299,6 +299,7 @@ class AdvancedSftpClientAgent(BaseAgent): def check_rfile_status(self, file_path:str): try: + print(f'checkin rfile status {file_path}') iobuf, dig = self.generate_rfile_hash(file_path, return_also_buffer=True) return { 'exists' : True, @@ -315,6 +316,7 @@ class AdvancedSftpClientAgent(BaseAgent): def check_rfolder_status(self, path:str): try: + print(f'checkin rfolder status {path}') dig = self.generate_tree_hash_oversftp(path) return { 'exists' : True,