async code

src/client/agent.py

@@ -65,5 +65,5 @@ class SyncAgent():
       print('name: %s is regular FILE' % (attr.filename))
 
 
-synca = _agent.SyncAgent()
+synca = SyncAgent()
 # sftpc = a.get_sftp_client()

src/snapshot/generate/remote.py

@@ -4,6 +4,9 @@ import hashlib
 import json
 import gzip
 
+import asyncio
+import functools
+
 from client import agent as _agent
 
 
@@ -56,6 +59,65 @@ def generate_tree_hash_oversftp(root_path :str):
 
   return rtreemap
 
+async def generate_rfile_hash_async(file_path, hexdigest=True, client=None):
+  if not client:
+    a = _agent.synca
+    client = a.get_sftp_client()
+
+  with client.open(file_path, "rb") as f:
+    buf = f.read()
+
+  if hexdigest: return hashlib.md5(buf).hexdigest()
+  return hashlib.md5(buf).digest()
+
+async def generate_tree_hash_oversftp_async(root_path :str):
+  '''
+    @param             root_path        string, root_path in remote server
+    generate a map of hashes starting from `root_path` recursively
+  '''
+  # if not os.path.isdir(root_path):
+  #   raise Exception('Provide a valid folder to start the hashing')
+  #
+  # if not root_path.endswith(os.path.sep):
+  #   root_path = root_path + os.path.sep
+  # root_path = check_isdir(root_path)
+
+  rtreemap = {}
+
+  if not root_path.endswith(os.path.sep):
+    root_path = root_path + os.path.sep
+
+  a = _agent.synca
+  sftpc = a.get_sftp_client()
+
+  def dtask_done_cback(fname, f):
+    print('dtask done %s' % (fname,))
+    rtreemap[fname] = f.result()
+
+  def ftask_done_cback(fname, f):
+    print('ftask done %s' % (fname,))
+    rtreemap[fname] = f.result()
+
+  # futures_map = {}
+  tasks = []
+  for item in sftpc.listdir_attr(root_path):
+    absolute_item_path = root_path + item.filename
+    print('absolute_item_path: %s, item %s, isdir: %s' % (absolute_item_path, item.filename, stat.S_ISDIR(item.st_mode)))
+    if stat.S_ISDIR(item.st_mode):
+      # rtreemap[item.filename] = await generate_tree_hash_oversftp_async(absolute_item_path)
+      dtask = asyncio.create_task(generate_tree_hash_oversftp_async(absolute_item_path))
+      dtask.add_done_callback(functools.partial(dtask_done_cback, item.filename))
+      tasks.append(dtask)
+    else:
+      # rtreemap[item.filename] = await generate_rfile_hash_async(absolute_item_path, client=sftpc)
+      ftask = asyncio.create_task(generate_rfile_hash_async(absolute_item_path, client=sftpc))
+      ftask.add_done_callback(functools.partial(ftask_done_cback, item.filename))
+      tasks.append(ftask)
+      # item.filename
+  
+  await asyncio.gather(*tasks)
+  return rtreemap
+
 
 '''
 a