notanamber/masy
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

agent sync

Browse Source
This commit is contained in:
Amber 2024-10-11 16:31:57 +02:00
parent 6b6cdf08b8
commit 8106b17ee4
4 changed files with 76 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

132
src/iface/sync_manager.py
View File

@ -157,6 +157,11 @@ class Manager():
remote_file_path = remote_file_path.replace('.%s' % (os.sep), self.remote_path)
return remote_file_path
def get_remote_mount_point(self, node):
node_path = node['rel_path']
remote_mount_point = node_path.replace('.%s' % (os.sep), self.remote_path)
return remote_mount_point
def add_sync_to_list(self):
home_dir = os.environ.get('HOME')
home_dir = self.normalize_path(home_dir)
@ -257,7 +262,9 @@ class Manager():
'''
absolute_remote_path = self.get_absolute_remote_path(node)
agent = self.get_agent()
node_type = node['last_type']
## if node not exist in the local tree there is not last_type
## we use cur_type (is this correct?)
node_type = node.get('last_type') or node['cur_type']
if node_type == 'f':
return agent.check_rfile_status(absolute_remote_path)
## node is a folder
@ -273,11 +280,12 @@ class Manager():
agent = self.get_agent()
if node['cur_type'] == 'f':
## simply put
return agent.copy_file_to_remote(absolute_local_path, absolute_remote_path)
return agent.copy_file(absolute_local_path, absolute_remote_path)
elif node['cur_type'] == 'd':
#cur_hash contains the entire subtree to recreate on remote
## it is better to remove the root node if already exists on remote
agent.remove_dir(absolute_remote_path)
remote_mount_point = self.get_remote_mount_point(node)
return agent.copy_dir(absolute_local_path, remote_mount_point)
def show_conflicts(self):
@ -333,7 +341,53 @@ class Manager():
print(f'force local node: {unmerged_local_path} to remote path: {absolute_remote_path}')
self.copy_node_to_remote(node)
def sync_added(self, added=False):
if added is None:
local_snap_diff = self.get_local_snap_diff()
changes = local_snap_diff.get('added') or []
for node in added:
node_name = node['name']
last_hash = node.get('last_hash')
if last_hash is not None:
raise Exception('node added has a last hash!!')
print(f'Checking local {node_name}')
rstatus = self.get_rnode_status(node)
rfile_buf, rhash, rexists = map(rstatus.get, ['iobuffer', 'hash', 'exists'])
if rexists:
raise Exception('This case must be managed!!! remote node appeared but in the local tree there was not')
self.copy_node_to_remote(node)
def sync_changed(self, changes=None):
if changes is None:
local_snap_diff = self.get_local_snap_diff()
changes = local_snap_diff.get('changed') or []
unmerged = []
for node in changes:
node_name = node['name']
last_hash = node['last_hash']
print(f'Checking local {node_name}')
rstatus = self.get_rnode_status(node)
rfile_buf, rhash, rexists = map(rstatus.get, ['iobuffer', 'hash', 'exists'])
node['remote_hash'] = rhash
node['remote_file_buf'] = rfile_buf
if not rexists or last_hash != rhash:
print(f'Put node {node_name} in unmerged, node not exists in remote or is changed from the last local snapshot')
unmerged.append(node)
continue
print(f'You can proceed to push {node_name} file it is not changed from the last version')
self.copy_node_to_remote(node)
if unmerged:
self.store_unmerged_diff(unmerged)
def sync(self):
'''
@ -351,78 +405,12 @@ class Manager():
# rhagent = self.init_rh_agent()
# agent = self.get_agent()
changes = local_snap_diff.get('changed') or []
self.sync_changed(changes=changes)
unmerged = []
added = local_snap_diff.get('added') or []
self.sync_added(added=added)
for node in changes:
node_name = node['name']
last_hash = node['last_hash']
print(f'Checking local {node_name}')
rstatus = self.get_rnode_status(node)
rfile_buf, rhash, rexists = map(rstatus.get, ['iobuffer', 'hash', 'exists'])
node['remote_hash'] = rhash
node['remote_file_buf'] = rfile_buf
if not rexists or last_hash != rhash:
print(f'Put node {node_name} in unmerged, node not exists in remote or is changed from the last local snapshot')
unmerged.append(node)
else:
print(f'You can proceed to push {node_name} file it is not changed from the last version')
if unmerged:
self.store_unmerged_diff(unmerged)
return
# for node in changes:
# node_name = node['name']
# node_path = node['path']
# node_current_type = node['current_type']
# node_last_type = node['last_type']
# node_current_hash = node['cur_hash']
# node_last_hash = node['last_hash']
#
# if not (node_current_type == node_last_type):
# print(f'node {node_name} change type in local tree')
# if node_last_type == 'f':
# # remote_file_path = '%s%s' % (node_path, node_name)
# # remote_file_path = remote_file_path.replace('.%s' % (os.sep), self.remote_path)
# remote_file_path = self.get_absolute_remote_path(node)
# rfile_buf, rhash = agent.generate_file_hash_oversftp(remote_file_path, return_also_buffer=True)
# if node_last_hash == rhash:
# print(f'You can proceed to push {node_name} file it is not changed from the last version')
# # local_file_path = '%s%s' % (node_path, node_name)
# # local_file_path = local_file_path.replace('.%s' % (os.sep), self.local_path)
# local_file_path = self.get_absolute_local_path(node)
# agent.put(local_file_path, remote_file_path, lambda x,y: print(f'{local_file_path} copied correctly to remote'))
# else:
# print(f'{node_path} file it changed local hash: {node_last_hash}, remote hash {rhash}, you can\'t push it directly')
# # self.store_unmerged_diff(node)
# node['remote_hash'] = rhash
# node['remote_file'] = rfile_buf
# unmerged.append(node)
#
# if node_last_type == 'd':
# # remote_path = '%s%s' % (node_path, node_name)
# # remote_path = remote_file_path.replace('.%s' % (os.sep), self.remote_path)
# remote_path = self.get_absolute_local_path(node)
# rhash = agent.generate_tree_hash_oversftp(remote_path)
# if node_last_hash == rhash:
# print(f'You can proceed to push {node_name} folder it is not changed from the last version')
# else:
# print(f'{node_name} folder it changed from the last version, you can\'t push it directly')
#
# if unmerged:
# self.store_unmerged_diff(unmerged)
#
# ## managing added
# added = local_snap_diff.get('added') or []
# for node in added:
# ## if node not exists in remote copy it recursively in remote
# remote_file_path = self.get_absolute_remote_path(node)
# agent.copy(node)
#
# return local_snap_diff

14
src/lib/diff/fdiff.py
View File

@ -11,6 +11,7 @@ def is_buf(f):
return False
def get_lines(file):
if not file: return []
if is_buf(file):
return file.readlines()
with open(file, 'r') as f:
@ -21,8 +22,16 @@ def compute_diff(filea, fileb):
'''
file* can be a path or a buffer
'''
alines = get_lines(filea)
blines = get_lines(fileb)
try:
alines = get_lines(filea)
except:
print('alines error: file could be a binary')
alines = ['binary content ---\n']
try:
blines = get_lines(fileb)
except UnicodeDecodeError as e:
print('blines error: file could be a binary')
blines = ['--- binary content\n']
differ = difflib.Differ()
res_diff = list(differ.compare(alines, blines))
@ -47,7 +56,6 @@ def format_diff(filea, fileb, remove_diff_letters_code=False, filea_block_tag=''
"? " line not present in either input file -> to skip it
'''
filea_block_tag = get_file_block_tag(filea, filea_block_tag)
fileb_block_tag = get_file_block_tag(fileb, fileb_block_tag)

4
src/lib/repr/node.py
View File

@ -18,6 +18,8 @@ class Node:
@param last_type is the last type received from server
@param cur_type is the current type in local tree
@param last_hash is the last hash received from server (a directory can have or empty hash or a subtree)
@param cur_hash is the current hash in local tree
@param cur_hash is the current hash in local tree (in local the hash has changed from `last_hash` to `cur_hash`)
@param remote_hash is the remote hash (usually computed with a remote call)
no conflict -> `remote_hash` == `last_hash`
conflict -> `remote_hash` != `last_hash` this means we have lost a version of node
'''

2
src/lib/sclient/agent.py
View File

@ -299,6 +299,7 @@ class AdvancedSftpClientAgent(BaseAgent):
def check_rfile_status(self, file_path:str):
try:
print(f'checkin rfile status {file_path}')
iobuf, dig = self.generate_rfile_hash(file_path, return_also_buffer=True)
return {
'exists' : True,
@ -315,6 +316,7 @@ class AdvancedSftpClientAgent(BaseAgent):
def check_rfolder_status(self, path:str):
try:
print(f'checkin rfolder status {path}')
dig = self.generate_tree_hash_oversftp(path)
return {
'exists' : True,