agent sync
This commit is contained in:
parent
6b6cdf08b8
commit
8106b17ee4
@ -157,6 +157,11 @@ class Manager():
|
||||
remote_file_path = remote_file_path.replace('.%s' % (os.sep), self.remote_path)
|
||||
return remote_file_path
|
||||
|
||||
def get_remote_mount_point(self, node):
|
||||
node_path = node['rel_path']
|
||||
remote_mount_point = node_path.replace('.%s' % (os.sep), self.remote_path)
|
||||
return remote_mount_point
|
||||
|
||||
def add_sync_to_list(self):
|
||||
home_dir = os.environ.get('HOME')
|
||||
home_dir = self.normalize_path(home_dir)
|
||||
@ -257,7 +262,9 @@ class Manager():
|
||||
'''
|
||||
absolute_remote_path = self.get_absolute_remote_path(node)
|
||||
agent = self.get_agent()
|
||||
node_type = node['last_type']
|
||||
## if node not exist in the local tree there is not last_type
|
||||
## we use cur_type (is this correct?)
|
||||
node_type = node.get('last_type') or node['cur_type']
|
||||
if node_type == 'f':
|
||||
return agent.check_rfile_status(absolute_remote_path)
|
||||
## node is a folder
|
||||
@ -273,11 +280,12 @@ class Manager():
|
||||
agent = self.get_agent()
|
||||
if node['cur_type'] == 'f':
|
||||
## simply put
|
||||
return agent.copy_file_to_remote(absolute_local_path, absolute_remote_path)
|
||||
return agent.copy_file(absolute_local_path, absolute_remote_path)
|
||||
elif node['cur_type'] == 'd':
|
||||
#cur_hash contains the entire subtree to recreate on remote
|
||||
## it is better to remove the root node if already exists on remote
|
||||
agent.remove_dir(absolute_remote_path)
|
||||
remote_mount_point = self.get_remote_mount_point(node)
|
||||
return agent.copy_dir(absolute_local_path, remote_mount_point)
|
||||
|
||||
|
||||
def show_conflicts(self):
|
||||
@ -333,23 +341,29 @@ class Manager():
|
||||
print(f'force local node: {unmerged_local_path} to remote path: {absolute_remote_path}')
|
||||
self.copy_node_to_remote(node)
|
||||
|
||||
|
||||
|
||||
def sync(self):
|
||||
'''
|
||||
sync:
|
||||
- calculate local diff
|
||||
- collect necessary action to sync server with this node
|
||||
- do a remote diff and compare with the last version in this node
|
||||
if there are not differencies simply do necessary actions to sync
|
||||
if there is differences put the file diff in a specific folder
|
||||
'''
|
||||
# self.init_rh_agent()
|
||||
# last_tree = self.load_local_hash()
|
||||
# current_tree = self.compute_local_hash()
|
||||
def sync_added(self, added=False):
|
||||
if added is None:
|
||||
local_snap_diff = self.get_local_snap_diff()
|
||||
changes = local_snap_diff.get('added') or []
|
||||
|
||||
for node in added:
|
||||
node_name = node['name']
|
||||
last_hash = node.get('last_hash')
|
||||
if last_hash is not None:
|
||||
raise Exception('node added has a last hash!!')
|
||||
print(f'Checking local {node_name}')
|
||||
|
||||
rstatus = self.get_rnode_status(node)
|
||||
rfile_buf, rhash, rexists = map(rstatus.get, ['iobuffer', 'hash', 'exists'])
|
||||
if rexists:
|
||||
raise Exception('This case must be managed!!! remote node appeared but in the local tree there was not')
|
||||
|
||||
self.copy_node_to_remote(node)
|
||||
|
||||
|
||||
def sync_changed(self, changes=None):
|
||||
if changes is None:
|
||||
local_snap_diff = self.get_local_snap_diff()
|
||||
# rhagent = self.init_rh_agent()
|
||||
# agent = self.get_agent()
|
||||
changes = local_snap_diff.get('changed') or []
|
||||
|
||||
unmerged = []
|
||||
@ -367,62 +381,36 @@ class Manager():
|
||||
if not rexists or last_hash != rhash:
|
||||
print(f'Put node {node_name} in unmerged, node not exists in remote or is changed from the last local snapshot')
|
||||
unmerged.append(node)
|
||||
else:
|
||||
continue
|
||||
|
||||
print(f'You can proceed to push {node_name} file it is not changed from the last version')
|
||||
self.copy_node_to_remote(node)
|
||||
|
||||
if unmerged:
|
||||
self.store_unmerged_diff(unmerged)
|
||||
|
||||
def sync(self):
|
||||
'''
|
||||
sync:
|
||||
- calculate local diff
|
||||
- collect necessary action to sync server with this node
|
||||
- do a remote diff and compare with the last version in this node
|
||||
if there are not differencies simply do necessary actions to sync
|
||||
if there is differences put the file diff in a specific folder
|
||||
'''
|
||||
# self.init_rh_agent()
|
||||
# last_tree = self.load_local_hash()
|
||||
# current_tree = self.compute_local_hash()
|
||||
local_snap_diff = self.get_local_snap_diff()
|
||||
# rhagent = self.init_rh_agent()
|
||||
# agent = self.get_agent()
|
||||
changes = local_snap_diff.get('changed') or []
|
||||
self.sync_changed(changes=changes)
|
||||
|
||||
added = local_snap_diff.get('added') or []
|
||||
self.sync_added(added=added)
|
||||
|
||||
|
||||
return
|
||||
|
||||
|
||||
# for node in changes:
|
||||
# node_name = node['name']
|
||||
# node_path = node['path']
|
||||
# node_current_type = node['current_type']
|
||||
# node_last_type = node['last_type']
|
||||
# node_current_hash = node['cur_hash']
|
||||
# node_last_hash = node['last_hash']
|
||||
#
|
||||
# if not (node_current_type == node_last_type):
|
||||
# print(f'node {node_name} change type in local tree')
|
||||
# if node_last_type == 'f':
|
||||
# # remote_file_path = '%s%s' % (node_path, node_name)
|
||||
# # remote_file_path = remote_file_path.replace('.%s' % (os.sep), self.remote_path)
|
||||
# remote_file_path = self.get_absolute_remote_path(node)
|
||||
# rfile_buf, rhash = agent.generate_file_hash_oversftp(remote_file_path, return_also_buffer=True)
|
||||
# if node_last_hash == rhash:
|
||||
# print(f'You can proceed to push {node_name} file it is not changed from the last version')
|
||||
# # local_file_path = '%s%s' % (node_path, node_name)
|
||||
# # local_file_path = local_file_path.replace('.%s' % (os.sep), self.local_path)
|
||||
# local_file_path = self.get_absolute_local_path(node)
|
||||
# agent.put(local_file_path, remote_file_path, lambda x,y: print(f'{local_file_path} copied correctly to remote'))
|
||||
# else:
|
||||
# print(f'{node_path} file it changed local hash: {node_last_hash}, remote hash {rhash}, you can\'t push it directly')
|
||||
# # self.store_unmerged_diff(node)
|
||||
# node['remote_hash'] = rhash
|
||||
# node['remote_file'] = rfile_buf
|
||||
# unmerged.append(node)
|
||||
#
|
||||
# if node_last_type == 'd':
|
||||
# # remote_path = '%s%s' % (node_path, node_name)
|
||||
# # remote_path = remote_file_path.replace('.%s' % (os.sep), self.remote_path)
|
||||
# remote_path = self.get_absolute_local_path(node)
|
||||
# rhash = agent.generate_tree_hash_oversftp(remote_path)
|
||||
# if node_last_hash == rhash:
|
||||
# print(f'You can proceed to push {node_name} folder it is not changed from the last version')
|
||||
# else:
|
||||
# print(f'{node_name} folder it changed from the last version, you can\'t push it directly')
|
||||
#
|
||||
# if unmerged:
|
||||
# self.store_unmerged_diff(unmerged)
|
||||
#
|
||||
# ## managing added
|
||||
# added = local_snap_diff.get('added') or []
|
||||
# for node in added:
|
||||
# ## if node not exists in remote copy it recursively in remote
|
||||
# remote_file_path = self.get_absolute_remote_path(node)
|
||||
# agent.copy(node)
|
||||
#
|
||||
# return local_snap_diff
|
||||
|
||||
|
@ -11,6 +11,7 @@ def is_buf(f):
|
||||
return False
|
||||
|
||||
def get_lines(file):
|
||||
if not file: return []
|
||||
if is_buf(file):
|
||||
return file.readlines()
|
||||
with open(file, 'r') as f:
|
||||
@ -21,8 +22,16 @@ def compute_diff(filea, fileb):
|
||||
'''
|
||||
file* can be a path or a buffer
|
||||
'''
|
||||
try:
|
||||
alines = get_lines(filea)
|
||||
except:
|
||||
print('alines error: file could be a binary')
|
||||
alines = ['binary content ---\n']
|
||||
try:
|
||||
blines = get_lines(fileb)
|
||||
except UnicodeDecodeError as e:
|
||||
print('blines error: file could be a binary')
|
||||
blines = ['--- binary content\n']
|
||||
|
||||
differ = difflib.Differ()
|
||||
res_diff = list(differ.compare(alines, blines))
|
||||
@ -47,7 +56,6 @@ def format_diff(filea, fileb, remove_diff_letters_code=False, filea_block_tag=''
|
||||
"? " line not present in either input file -> to skip it
|
||||
'''
|
||||
|
||||
|
||||
filea_block_tag = get_file_block_tag(filea, filea_block_tag)
|
||||
fileb_block_tag = get_file_block_tag(fileb, fileb_block_tag)
|
||||
|
||||
|
@ -18,6 +18,8 @@ class Node:
|
||||
@param last_type is the last type received from server
|
||||
@param cur_type is the current type in local tree
|
||||
@param last_hash is the last hash received from server (a directory can have or empty hash or a subtree)
|
||||
@param cur_hash is the current hash in local tree
|
||||
@param cur_hash is the current hash in local tree (in local the hash has changed from `last_hash` to `cur_hash`)
|
||||
@param remote_hash is the remote hash (usually computed with a remote call)
|
||||
no conflict -> `remote_hash` == `last_hash`
|
||||
conflict -> `remote_hash` != `last_hash` this means we have lost a version of node
|
||||
'''
|
||||
|
@ -299,6 +299,7 @@ class AdvancedSftpClientAgent(BaseAgent):
|
||||
|
||||
def check_rfile_status(self, file_path:str):
|
||||
try:
|
||||
print(f'checkin rfile status {file_path}')
|
||||
iobuf, dig = self.generate_rfile_hash(file_path, return_also_buffer=True)
|
||||
return {
|
||||
'exists' : True,
|
||||
@ -315,6 +316,7 @@ class AdvancedSftpClientAgent(BaseAgent):
|
||||
|
||||
def check_rfolder_status(self, path:str):
|
||||
try:
|
||||
print(f'checkin rfolder status {path}')
|
||||
dig = self.generate_tree_hash_oversftp(path)
|
||||
return {
|
||||
'exists' : True,
|
||||
|
Loading…
Reference in New Issue
Block a user