More work on bridging, first WIP Web backend

LibWinDog/Platforms/Matrix/Matrix.py

@@ -23,7 +23,7 @@ import nio
 import queue
 
 MatrixClient = None
-MatrixQueue = []#queue.Queue()
+MatrixQueue = queue.Queue()
 
 def MatrixMain() -> bool:
 	if not (MatrixUrl and MatrixUsername and (MatrixPassword or MatrixToken)):
@@ -33,11 +33,10 @@ def MatrixMain() -> bool:
 		MatrixUsername = new
 	async def queue_handler():
 		asyncio.ensure_future(queue_handler())
-		if not len(MatrixQueue):
+		try:
-			# avoid 100% CPU usage ☠️
+			MatrixSender(*MatrixQueue.get(block=False))
-			time.sleep(0.01)
+		except queue.Empty:
-		while len(MatrixQueue):
+			time.sleep(0.01) # avoid 100% CPU usage ☠️
-			MatrixSender(*MatrixQueue.pop(0))
 	async def client_main() -> None:
 		global MatrixClient
 		MatrixClient = nio.AsyncClient(MatrixUrl, MatrixUsername)
@@ -69,6 +68,9 @@ def MatrixMakeInputMessageData(room:nio.MatrixRoom, event:nio.RoomMessage) -> In
 			#name = , # TODO name must be get via a separate API request (and so maybe we should cache it)
 		),
 	)
+	if (mxc_url := ObjGet(data, "media.url")) and mxc_url.startswith("mxc://"):
+		_, _, server_name, media_id = mxc_url.split('/')
+		data.media["url"] = ("https://" + server_name + nio.Api.download(server_name, media_id)[1])
 	data.command = ParseCommand(data.text_plain)
 	data.user.settings = (GetUserSettings(data.user.id) or SafeNamespace())
 	return data
@@ -85,7 +87,7 @@ def MatrixSender(context:EventContext, data:OutputMessageData):
 	try:
 		asyncio.get_event_loop()
 	except RuntimeError:
-		MatrixQueue.append((context, data))
+		MatrixQueue.put((context, data))
 		return None
 	asyncio.create_task(context.manager.room_send(
 		room_id=(data.room_id or ObjGet(context, "event.room.room_id")),

LibWinDog/Platforms/Telegram/Telegram.py

@@ -72,6 +72,7 @@ def TelegramHandler(update:telegram.Update, context:CallbackContext=None) -> Non
 
 def TelegramSender(context:EventContext, data:OutputMessageData):
 	result = None
+	# TODO clean this
 	if data.room_id:
 		result = context.manager.bot.send_message(data.room_id, text=data.text_plain)
 	else:

LibWinDog/Platforms/Web.py

@@ -1,13 +0,0 @@
-# ================================== #
-# WinDog multi-purpose chatbot       #
-# Licensed under AGPLv3 by OctoSpacc #
-# ================================== #
-
-def WebMain() -> None:
-	pass
-
-def WebSender() -> None:
-	pass
-
-#RegisterPlatform(name="Web", main=WebMain, sender=WebSender)
-

LibWinDog/Platforms/Web/Web.py

@@ -0,0 +1,86 @@
+# ================================== #
+# WinDog multi-purpose chatbot       #
+# Licensed under AGPLv3 by OctoSpacc #
+# ================================== #
+
+""" # windog config start # """
+
+WebConfig = {
+	"host": ("0.0.0.0", 30264),
+	"url": "https://windog.octt.eu.org",
+}
+
+""" # end windog config # """
+
+import queue
+from http.server import BaseHTTPRequestHandler
+from LibWinDog.Platforms.Web.multithread_http_server import MultiThreadHttpServer
+
+WebQueues = {}
+
+default_css = """<style>
+	* { box-sizing: border-box; }
+	iframe { width: 100%; height: 3em; top: 0; position: sticky; }
+	textarea { width: calc(100% - 3em); height: 2em; }
+	input { width: 2em; }
+	</style>"""
+
+class WebServerClass(BaseHTTPRequestHandler):
+	def do_GET(self):
+		if self.path == '/':
+			uuid = str(time.time())
+			WebQueues[uuid] = queue.Queue()
+			self.send_response(302)
+			self.send_header("Location", f"/{uuid}")
+			self.end_headers()
+			return
+		uuid = self.path.split('/')[-1]
+		if self.path.startswith("/form/"):
+			self.send_response(200)
+			self.send_header("Content-Type", "text/html; charset=UTF-8")
+			self.end_headers()
+			self.wfile.write(f'{default_css}<form method="POST" action="/form/{uuid}"><textarea name="text"></textarea><input type="submit" value="📤️"/></form>'.encode())
+			return
+		self.send_response(200)
+		self.send_header("Content-Type", "text/html; charset=UTF-8")
+		self.send_header("Content-Encoding", "chunked")
+		self.end_headers()
+		self.wfile.write(f'{default_css}<h3><a href="/">WinDog</a></h3><iframe src="/form/{uuid}"></iframe>'.encode())
+		while True: # this apparently makes us lose threads and the web bot becomes unusable, we should handle dropped connections
+			try:
+				self.wfile.write(("<p>" + WebQueues[uuid].get(block=False).text_html + "</p>").encode())
+			except queue.Empty:
+				time.sleep(0.01)
+
+	def do_POST(self):
+		uuid = self.path.split('/')[-1]
+		text = urlparse.unquote_plus(self.rfile.read(int(self.headers["Content-Length"])).decode().split('=')[1])
+		self.send_response(302)
+		self.send_header("Location", f"/form/{uuid}")
+		self.end_headers()
+		data = WebMakeInputMessageData(text, uuid)
+		OnMessageParsed(data)
+		if (command := ObjGet(data, "command.name")):
+			CallEndpoint(command, EventContext(platform="web", event=SafeNamespace(room_id=uuid)), data)
+
+def WebMakeInputMessageData(text:str, uuid:str) -> InputMessageData:
+	return InputMessageData(
+		text_plain = text,
+		command = ParseCommand(text),
+		room = SafeNamespace(
+			id = f"web:{uuid}",
+		),
+		user = SafeNamespace(
+			settings = SafeNamespace(),
+		),
+	)
+
+def WebMain() -> None:
+	server = MultiThreadHttpServer(WebConfig["host"], 32, WebServerClass)
+	server.start(background=True)
+
+def WebSender(context:EventContext, data:OutputMessageData) -> None:
+	WebQueues[context.event.room_id].put(data)
+
+RegisterPlatform(name="Web", main=WebMain, sender=WebSender)
+

LibWinDog/Platforms/Web/multithread_http_server.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+"""
+MIT License
+
+Copyright (c) 2018 Ortis (cao.ortis.org@gmail.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+
+import socket
+import threading
+import time
+from http.server import HTTPServer
+import logging
+
+
+class MultiThreadHttpServer:
+
+	def __init__(self, host, parallelism, http_handler_class, request_callback=None, log=None):
+		"""
+		:param host: host to bind. example: '127.0.0.1:80'
+		:param parallelism: number of thread listener and backlog
+		:param http_handler_class: the handler class extending BaseHTTPRequestHandler
+		:param request_callback: callback on incoming request. This method can be accede in the HTTPHandler instance.
+				Example: self.server.request_callback(
+														'GET',  # specify http method
+														self  # pass the HTTPHandler instance
+													)
+		"""
+
+		self.host = host
+		self.parallelism = parallelism
+		self.http_handler_class = http_handler_class
+		self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+		self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+		self.request_callback = request_callback
+		self.connection_handlers = []
+		self.stop_requested = False
+		self.log = log
+
+	def start(self, background=False):
+		self.socket.bind(self.host)
+		self.socket.listen(self.parallelism)
+
+		if self.log is not None:
+			self.log.debug("Creating "+str(self.parallelism)+" connection handler")
+
+		for i in range(self.parallelism):
+			ch = ConnectionHandler(self.socket, self.http_handler_class, self.request_callback)
+			ch.start()
+			self.connection_handlers.append(ch)
+
+		if background:
+			if self.log is not None:
+				self.log.debug("Serving (background thread)")
+			threading.Thread(target=self.__serve).start()
+		else:
+			if self.log is not None:
+				self.log.debug("Serving (current thread)")
+			self.__serve()
+
+	def stop(self):
+		self.stop_requested = True
+		for ch in self.connection_handlers:
+			ch.stop()
+
+	def __serve(self):
+		"""
+		Serve until stop() is called. Blocking method
+		:return:
+		"""
+		while not self.stop_requested:
+			time.sleep(1)
+
+
+class ConnectionHandler(threading.Thread, HTTPServer):
+
+	def __init__(self, sock, http_handler_class, request_callback=None):
+		HTTPServer.__init__(self, sock.getsockname(), http_handler_class, False)
+		self.socket = sock
+		self.server_bind = self.server_close = lambda self: None
+		self.HTTPHandler = http_handler_class
+		self.request_callback = request_callback
+
+		threading.Thread.__init__(self)
+		self.daemon = True
+		self.stop_requested = False
+
+	def stop(self):
+		self.stop_requested = True
+
+	def run(self):
+		""" Each thread process request forever"""
+		self.serve_forever()
+
+	def serve_forever(self):
+		""" Handle requests until stopped """
+		while not self.stop_requested:
+			self.handle_request()
+
+		print("Finish" + str(threading.current_thread()))
+

LibWinDog/Types.py

@@ -12,12 +12,17 @@ class SafeNamespace(SimpleNamespace):
 		except AttributeError:
 			return None
 
+# we just use these for type hinting:
+
 class EventContext(SafeNamespace):
 	pass
 
-class InputMessageData(SafeNamespace):
+class MessageData(SafeNamespace):
 	pass
 
-class OutputMessageData(SafeNamespace):
+class InputMessageData(MessageData):
+	pass
+
+class OutputMessageData(MessageData):
 	pass
 

ModWinDog/Echo/Echo.py

@@ -5,7 +5,7 @@
 
 def cEcho(context:EventContext, data:InputMessageData) -> None:
 	if not (text := ObjGet(data, "command.body")):
-		return SendMessage(context, OutputMessageData(text_html=context.endpoint.get_string("empty", data.user.settings.language)))
+		return SendMessage(context, {"text_html": context.endpoint.get_string("empty", data.user.settings.language)})
 	prefix = f'<a href="{data.message_url}">🗣️</a> '
 	#prefix = f"[🗣️]({context.linker(data).message}) "
 	if len(data.command.tokens) == 2:
@@ -17,7 +17,7 @@ def cEcho(context:EventContext, data:InputMessageData) -> None:
 		if nonascii:
 			# text is not ascii, probably an emoji (altough not necessarily), so just pass as is (useful for Telegram emojis)
 			prefix = ''
-	SendMessage(context, OutputMessageData(text_html=(prefix + html_escape(text))))
+	SendMessage(context, {"text_html": (prefix + html_escape(text))})
 
 RegisterModule(name="Echo", endpoints=[
 	SafeNamespace(names=["echo"], handler=cEcho),

ModWinDog/Internet/Internet.py

@@ -19,7 +19,7 @@ def cEmbedded(context:EventContext, data:InputMessageData) -> None:
 	if len(data.command.tokens) >= 2:
 		# Find links in command body
 		text = (data.text_markdown + ' ' + data.text_plain)
-	elif (quoted := data.quoted) and (quoted.text_auto or quoted.text_markdown or quoted.text_html):
+	elif (quoted := data.quoted) and (quoted.text_plain or quoted.text_markdown or quoted.text_html):
 		# Find links in quoted message
 		text = ((quoted.text_markdown or '') + ' ' + (quoted.text_plain or '') + ' ' + (quoted.text_html or ''))
 	else:

ModWinDog/Scrapers/Scrapers.py

@@ -107,8 +107,8 @@ def cCraiyonSelenium(context:EventContext, data:InputMessageData) -> None:
 			for img_elem in img_list:
 				img_array.append({"url": img_elem.get_attribute("src")}) #, "bytes": HttpReq(img_url).read()})
 			SendMessage(context, {
-				"TextPlain": f'"{prompt}"',
+				"text_plain": f'"{prompt}"',
-				"TextMarkdown": (f'"_{CharEscape(prompt, "MARKDOWN")}_"'),
+				"text_html": f'"<i>{html_escape(prompt)}</i>"',
 				"media": img_array,
 			})
 			return closeSelenium(driver_index, driver)

WinDog.py

@@ -14,7 +14,7 @@ from os.path import isfile, isdir
 from random import choice, choice as randchoice, randint
 from threading import Thread
 from traceback import format_exc, format_exc as traceback_format_exc
-from urllib import parse as urlparse, urllib_parse
+from urllib import parse as urlparse, parse as urllib_parse
 from yaml import load as yaml_load, BaseLoader as yaml_BaseLoader
 from bs4 import BeautifulSoup
 from markdown import markdown
@@ -29,11 +29,14 @@ def ObjectUnion(*objects:object, clazz:object=None):
 	dikt = {}
 	auto_clazz = None
 	for obj in objects:
+		obj_clazz = obj.__class__
+		if not obj:
+			continue
 		if type(obj) == dict:
 			obj = (clazz or SafeNamespace)(**obj)
 		for key, value in tuple(obj.__dict__.items()):
 			dikt[key] = value
-		auto_clazz = obj.__class__
+		auto_clazz = obj_clazz
 	return (clazz or auto_clazz)(**dikt)
 
 def Log(text:str, level:str="?", *, newline:bool|None=None, inline:bool=False) -> None:
@@ -185,7 +188,7 @@ def ParseCommand(text:str) -> SafeNamespace|None:
 	command.body = text[len(command.tokens[0]):].strip()
 	if command.name not in Endpoints:
 		return command
-	if (endpoint_arguments := Endpoints[command.name].arguments):#["arguments"]):
+	if (endpoint_arguments := Endpoints[command.name].arguments):
 		command.arguments = {}
 		index = 1
 		for key in endpoint_arguments:
@@ -201,17 +204,35 @@ def ParseCommand(text:str) -> SafeNamespace|None:
 	return command
 
 def OnMessageParsed(data:InputMessageData) -> None:
-	DumpMessage(data)
+	dump_message(data, prefix='>')
-	UpdateUserDb(data.user)
+	#handle_bridging(SendMessage, data, from_sent=False)
-	for bridge in BridgesConfig:
+	update_user_db(data.user)
-		if data.room.id in bridge:
-			rooms = list(bridge)
-			rooms.remove(data.room.id)
-			for room in rooms:
-				tokens = room.split(':')
-				SendMessage(SafeNamespace(platform=tokens[0]), ObjectUnion(data, {"room_id": ':'.join(tokens)}))
 
-def UpdateUserDb(user:SafeNamespace) -> None:
+def OnMessageSent(data:OutputMessageData) -> None:
+	dump_message(data, prefix='<')
+	#handle_bridging(SendMessage, data, from_sent=True) # TODO fix duplicate messages lol
+
+# TODO: fix to send messages to different rooms, this overrides destination data but that gives problems with rebroadcasting the bot's own messages
+def handle_bridging(method:callable, data:MessageData, from_sent:bool):
+	if data.user:
+		if (text_plain := ObjGet(data, "text_plain")):
+			text_plain = f"<{data.user.name}>: {text_plain}"
+		if (text_html := ObjGet(data, "text_html")):
+			text_html = (urlparse.quote(f"<{data.user.name}>: ") + text_html)
+	for bridge in BridgesConfig:
+		if data.room.id not in bridge:
+			continue
+		rooms = list(bridge)
+		rooms.remove(data.room.id)
+		for room_id in rooms:
+			method(
+				SafeNamespace(platform=room_id.split(':')[0]),
+				ObjectUnion(data, {"room_id": room_id}, ({"text_plain": text_plain, "text_markdown": None, "text_html": text_html} if data.user else None)),
+				from_sent)
+
+def update_user_db(user:SafeNamespace) -> None:
+	if not (user and user.id):
+		return
 	try:
 		User.get(User.id == user.id)
 	except User.DoesNotExist:
@@ -222,17 +243,17 @@ def UpdateUserDb(user:SafeNamespace) -> None:
 		except User.DoesNotExist:
 			User.create(id=user.id, id_hash=user_hash)
 
-def DumpMessage(data:InputMessageData) -> None:
+def dump_message(data:InputMessageData, prefix:str='') -> None:
 	if not (Debug and (DumpToFile or DumpToConsole)):
 		return
 	text = (data.text_plain.replace('\n', '\\n') if data.text_plain else '')
-	text = f"[{int(time.time())}] [{time.ctime()}] [{data.room and data.room.id}] [{data.message_id}] [{data.user.id}] {text}"
+	text = f"{prefix} [{int(time.time())}] [{time.ctime()}] [{data.room and data.room.id}] [{data.message_id}] [{data.user and data.user.id}] {text}"
 	if DumpToConsole:
 		print(text, data)
 	if DumpToFile:
 		open((DumpToFile if (DumpToFile and type(DumpToFile) == str) else "./Dump.txt"), 'a').write(text + '\n')
 
-def SendMessage(context:EventContext, data:OutputMessageData) -> None:
+def SendMessage(context:EventContext, data:OutputMessageData, from_sent:bool=False) -> None:
 	data = (OutputMessageData(**data) if type(data) == dict else data)
 
 	# TODO remove this after all modules are changed
@@ -240,14 +261,16 @@ def SendMessage(context:EventContext, data:OutputMessageData) -> None:
 		data.text = data.Text
 	if data.TextPlain and not data.text_plain:
 		data.text_plain = data.TextPlain
-	if data.TextMarkdown and not data.text_markdown:
+	if data.text and not data.text_plain:
-		data.text_markdown = data.TextMarkdown
+		data.text_plain = data.text
 
 	if data.text_plain or data.text_markdown or data.text_html:
 		if data.text_html and not data.text_plain:
 			data.text_plain = BeautifulSoup(data.text_html, "html.parser").get_text()
 		elif data.text_markdown and not data.text_plain:
 			data.text_plain = data.text_markdown
+		elif data.text_plain and not data.text_html:
+			data.text_html = html_escape(data.text_plain)
 	elif data.text:
 		# our old system attempts to always receive Markdown and retransform when needed
 		data.text_plain = MdToTxt(data.text)
@@ -266,7 +289,10 @@ def SendMessage(context:EventContext, data:OutputMessageData) -> None:
 	platform = Platforms[context.platform]
 	if (not context.manager) and (manager := platform.manager_class):
 		context.manager = (manager() if callable(manager) else manager)
-	return platform.sender(context, data)
+	result = platform.sender(context, data)
+	if not from_sent:
+		OnMessageSent(data)
+	return result
 
 def SendNotice(context:EventContext, data) -> None:
 	pass
@@ -275,7 +301,7 @@ def DeleteMessage(context:EventContext, data) -> None:
 	pass
 
 def RegisterPlatform(name:str, main:callable, sender:callable, linker:callable=None, *, event_class=None, manager_class=None) -> None:
-	Platforms[name.lower()] = SafeNamespace(main=main, sender=sender, linker=linker, event_class=event_class, manager_class=manager_class)
+	Platforms[name.lower()] = SafeNamespace(name=name, main=main, sender=sender, linker=linker, event_class=event_class, manager_class=manager_class)
 	Log(f"{name}, ", inline=True)
 
 def RegisterModule(name:str, endpoints:dict, *, group:str|None=None) -> None:
@@ -317,9 +343,9 @@ def Main() -> None:
 	#SetupDb()
 	SetupLocales()
 	Log(f"📨️ Initializing Platforms... ", newline=False)
-	for platform in Platforms:
+	for platform in Platforms.values():
-		if Platforms[platform].main():
+		if platform.main():
-			Log(f"{platform}, ", inline=True)
+			Log(f"{platform.name}, ", inline=True)
 	Log("...Done. ✅️", inline=True, newline=True)
 	Log("🐶️ WinDog Ready!")
 	while True: