More work on bridging, first WIP Web backend

This commit is contained in:
2024-07-01 01:17:35 +02:00
parent 754e199526
commit 6a1a21027c
10 changed files with 276 additions and 50 deletions

View File

@ -23,7 +23,7 @@ import nio
import queue
MatrixClient = None
MatrixQueue = []#queue.Queue()
MatrixQueue = queue.Queue()
def MatrixMain() -> bool:
if not (MatrixUrl and MatrixUsername and (MatrixPassword or MatrixToken)):
@ -33,11 +33,10 @@ def MatrixMain() -> bool:
MatrixUsername = new
async def queue_handler():
asyncio.ensure_future(queue_handler())
if not len(MatrixQueue):
# avoid 100% CPU usage ☠️
time.sleep(0.01)
while len(MatrixQueue):
MatrixSender(*MatrixQueue.pop(0))
try:
MatrixSender(*MatrixQueue.get(block=False))
except queue.Empty:
time.sleep(0.01) # avoid 100% CPU usage ☠️
async def client_main() -> None:
global MatrixClient
MatrixClient = nio.AsyncClient(MatrixUrl, MatrixUsername)
@ -69,6 +68,9 @@ def MatrixMakeInputMessageData(room:nio.MatrixRoom, event:nio.RoomMessage) -> In
#name = , # TODO name must be get via a separate API request (and so maybe we should cache it)
),
)
if (mxc_url := ObjGet(data, "media.url")) and mxc_url.startswith("mxc://"):
_, _, server_name, media_id = mxc_url.split('/')
data.media["url"] = ("https://" + server_name + nio.Api.download(server_name, media_id)[1])
data.command = ParseCommand(data.text_plain)
data.user.settings = (GetUserSettings(data.user.id) or SafeNamespace())
return data
@ -85,7 +87,7 @@ def MatrixSender(context:EventContext, data:OutputMessageData):
try:
asyncio.get_event_loop()
except RuntimeError:
MatrixQueue.append((context, data))
MatrixQueue.put((context, data))
return None
asyncio.create_task(context.manager.room_send(
room_id=(data.room_id or ObjGet(context, "event.room.room_id")),

View File

@ -72,6 +72,7 @@ def TelegramHandler(update:telegram.Update, context:CallbackContext=None) -> Non
def TelegramSender(context:EventContext, data:OutputMessageData):
result = None
# TODO clean this
if data.room_id:
result = context.manager.bot.send_message(data.room_id, text=data.text_plain)
else:

View File

@ -1,13 +0,0 @@
# ================================== #
# WinDog multi-purpose chatbot #
# Licensed under AGPLv3 by OctoSpacc #
# ================================== #
def WebMain() -> None:
pass
def WebSender() -> None:
pass
#RegisterPlatform(name="Web", main=WebMain, sender=WebSender)

86
LibWinDog/Platforms/Web/Web.py Executable file
View File

@ -0,0 +1,86 @@
# ================================== #
# WinDog multi-purpose chatbot #
# Licensed under AGPLv3 by OctoSpacc #
# ================================== #
""" # windog config start # """
WebConfig = {
"host": ("0.0.0.0", 30264),
"url": "https://windog.octt.eu.org",
}
""" # end windog config # """
import queue
from http.server import BaseHTTPRequestHandler
from LibWinDog.Platforms.Web.multithread_http_server import MultiThreadHttpServer
WebQueues = {}
default_css = """<style>
* { box-sizing: border-box; }
iframe { width: 100%; height: 3em; top: 0; position: sticky; }
textarea { width: calc(100% - 3em); height: 2em; }
input { width: 2em; }
</style>"""
class WebServerClass(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == '/':
uuid = str(time.time())
WebQueues[uuid] = queue.Queue()
self.send_response(302)
self.send_header("Location", f"/{uuid}")
self.end_headers()
return
uuid = self.path.split('/')[-1]
if self.path.startswith("/form/"):
self.send_response(200)
self.send_header("Content-Type", "text/html; charset=UTF-8")
self.end_headers()
self.wfile.write(f'{default_css}<form method="POST" action="/form/{uuid}"><textarea name="text"></textarea><input type="submit" value="📤️"/></form>'.encode())
return
self.send_response(200)
self.send_header("Content-Type", "text/html; charset=UTF-8")
self.send_header("Content-Encoding", "chunked")
self.end_headers()
self.wfile.write(f'{default_css}<h3><a href="/">WinDog</a></h3><iframe src="/form/{uuid}"></iframe>'.encode())
while True: # this apparently makes us lose threads and the web bot becomes unusable, we should handle dropped connections
try:
self.wfile.write(("<p>" + WebQueues[uuid].get(block=False).text_html + "</p>").encode())
except queue.Empty:
time.sleep(0.01)
def do_POST(self):
uuid = self.path.split('/')[-1]
text = urlparse.unquote_plus(self.rfile.read(int(self.headers["Content-Length"])).decode().split('=')[1])
self.send_response(302)
self.send_header("Location", f"/form/{uuid}")
self.end_headers()
data = WebMakeInputMessageData(text, uuid)
OnMessageParsed(data)
if (command := ObjGet(data, "command.name")):
CallEndpoint(command, EventContext(platform="web", event=SafeNamespace(room_id=uuid)), data)
def WebMakeInputMessageData(text:str, uuid:str) -> InputMessageData:
return InputMessageData(
text_plain = text,
command = ParseCommand(text),
room = SafeNamespace(
id = f"web:{uuid}",
),
user = SafeNamespace(
settings = SafeNamespace(),
),
)
def WebMain() -> None:
server = MultiThreadHttpServer(WebConfig["host"], 32, WebServerClass)
server.start(background=True)
def WebSender(context:EventContext, data:OutputMessageData) -> None:
WebQueues[context.event.room_id].put(data)
RegisterPlatform(name="Web", main=WebMain, sender=WebSender)

View File

@ -0,0 +1,119 @@
#!/usr/bin/env python
"""
MIT License
Copyright (c) 2018 Ortis (cao.ortis.org@gmail.com)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
import socket
import threading
import time
from http.server import HTTPServer
import logging
class MultiThreadHttpServer:
def __init__(self, host, parallelism, http_handler_class, request_callback=None, log=None):
"""
:param host: host to bind. example: '127.0.0.1:80'
:param parallelism: number of thread listener and backlog
:param http_handler_class: the handler class extending BaseHTTPRequestHandler
:param request_callback: callback on incoming request. This method can be accede in the HTTPHandler instance.
Example: self.server.request_callback(
'GET', # specify http method
self # pass the HTTPHandler instance
)
"""
self.host = host
self.parallelism = parallelism
self.http_handler_class = http_handler_class
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.request_callback = request_callback
self.connection_handlers = []
self.stop_requested = False
self.log = log
def start(self, background=False):
self.socket.bind(self.host)
self.socket.listen(self.parallelism)
if self.log is not None:
self.log.debug("Creating "+str(self.parallelism)+" connection handler")
for i in range(self.parallelism):
ch = ConnectionHandler(self.socket, self.http_handler_class, self.request_callback)
ch.start()
self.connection_handlers.append(ch)
if background:
if self.log is not None:
self.log.debug("Serving (background thread)")
threading.Thread(target=self.__serve).start()
else:
if self.log is not None:
self.log.debug("Serving (current thread)")
self.__serve()
def stop(self):
self.stop_requested = True
for ch in self.connection_handlers:
ch.stop()
def __serve(self):
"""
Serve until stop() is called. Blocking method
:return:
"""
while not self.stop_requested:
time.sleep(1)
class ConnectionHandler(threading.Thread, HTTPServer):
def __init__(self, sock, http_handler_class, request_callback=None):
HTTPServer.__init__(self, sock.getsockname(), http_handler_class, False)
self.socket = sock
self.server_bind = self.server_close = lambda self: None
self.HTTPHandler = http_handler_class
self.request_callback = request_callback
threading.Thread.__init__(self)
self.daemon = True
self.stop_requested = False
def stop(self):
self.stop_requested = True
def run(self):
""" Each thread process request forever"""
self.serve_forever()
def serve_forever(self):
""" Handle requests until stopped """
while not self.stop_requested:
self.handle_request()
print("Finish" + str(threading.current_thread()))

View File

@ -12,12 +12,17 @@ class SafeNamespace(SimpleNamespace):
except AttributeError:
return None
# we just use these for type hinting:
class EventContext(SafeNamespace):
pass
class InputMessageData(SafeNamespace):
class MessageData(SafeNamespace):
pass
class OutputMessageData(SafeNamespace):
class InputMessageData(MessageData):
pass
class OutputMessageData(MessageData):
pass

View File

@ -5,7 +5,7 @@
def cEcho(context:EventContext, data:InputMessageData) -> None:
if not (text := ObjGet(data, "command.body")):
return SendMessage(context, OutputMessageData(text_html=context.endpoint.get_string("empty", data.user.settings.language)))
return SendMessage(context, {"text_html": context.endpoint.get_string("empty", data.user.settings.language)})
prefix = f'<a href="{data.message_url}">🗣️</a> '
#prefix = f"[🗣️]({context.linker(data).message}) "
if len(data.command.tokens) == 2:
@ -17,7 +17,7 @@ def cEcho(context:EventContext, data:InputMessageData) -> None:
if nonascii:
# text is not ascii, probably an emoji (altough not necessarily), so just pass as is (useful for Telegram emojis)
prefix = ''
SendMessage(context, OutputMessageData(text_html=(prefix + html_escape(text))))
SendMessage(context, {"text_html": (prefix + html_escape(text))})
RegisterModule(name="Echo", endpoints=[
SafeNamespace(names=["echo"], handler=cEcho),

View File

@ -19,7 +19,7 @@ def cEmbedded(context:EventContext, data:InputMessageData) -> None:
if len(data.command.tokens) >= 2:
# Find links in command body
text = (data.text_markdown + ' ' + data.text_plain)
elif (quoted := data.quoted) and (quoted.text_auto or quoted.text_markdown or quoted.text_html):
elif (quoted := data.quoted) and (quoted.text_plain or quoted.text_markdown or quoted.text_html):
# Find links in quoted message
text = ((quoted.text_markdown or '') + ' ' + (quoted.text_plain or '') + ' ' + (quoted.text_html or ''))
else:

View File

@ -107,8 +107,8 @@ def cCraiyonSelenium(context:EventContext, data:InputMessageData) -> None:
for img_elem in img_list:
img_array.append({"url": img_elem.get_attribute("src")}) #, "bytes": HttpReq(img_url).read()})
SendMessage(context, {
"TextPlain": f'"{prompt}"',
"TextMarkdown": (f'"_{CharEscape(prompt, "MARKDOWN")}_"'),
"text_plain": f'"{prompt}"',
"text_html": f'"<i>{html_escape(prompt)}</i>"',
"media": img_array,
})
return closeSelenium(driver_index, driver)

View File

@ -14,7 +14,7 @@ from os.path import isfile, isdir
from random import choice, choice as randchoice, randint
from threading import Thread
from traceback import format_exc, format_exc as traceback_format_exc
from urllib import parse as urlparse, urllib_parse
from urllib import parse as urlparse, parse as urllib_parse
from yaml import load as yaml_load, BaseLoader as yaml_BaseLoader
from bs4 import BeautifulSoup
from markdown import markdown
@ -29,11 +29,14 @@ def ObjectUnion(*objects:object, clazz:object=None):
dikt = {}
auto_clazz = None
for obj in objects:
obj_clazz = obj.__class__
if not obj:
continue
if type(obj) == dict:
obj = (clazz or SafeNamespace)(**obj)
for key, value in tuple(obj.__dict__.items()):
dikt[key] = value
auto_clazz = obj.__class__
auto_clazz = obj_clazz
return (clazz or auto_clazz)(**dikt)
def Log(text:str, level:str="?", *, newline:bool|None=None, inline:bool=False) -> None:
@ -185,7 +188,7 @@ def ParseCommand(text:str) -> SafeNamespace|None:
command.body = text[len(command.tokens[0]):].strip()
if command.name not in Endpoints:
return command
if (endpoint_arguments := Endpoints[command.name].arguments):#["arguments"]):
if (endpoint_arguments := Endpoints[command.name].arguments):
command.arguments = {}
index = 1
for key in endpoint_arguments:
@ -201,17 +204,35 @@ def ParseCommand(text:str) -> SafeNamespace|None:
return command
def OnMessageParsed(data:InputMessageData) -> None:
DumpMessage(data)
UpdateUserDb(data.user)
dump_message(data, prefix='>')
#handle_bridging(SendMessage, data, from_sent=False)
update_user_db(data.user)
def OnMessageSent(data:OutputMessageData) -> None:
dump_message(data, prefix='<')
#handle_bridging(SendMessage, data, from_sent=True) # TODO fix duplicate messages lol
# TODO: fix to send messages to different rooms, this overrides destination data but that gives problems with rebroadcasting the bot's own messages
def handle_bridging(method:callable, data:MessageData, from_sent:bool):
if data.user:
if (text_plain := ObjGet(data, "text_plain")):
text_plain = f"<{data.user.name}>: {text_plain}"
if (text_html := ObjGet(data, "text_html")):
text_html = (urlparse.quote(f"<{data.user.name}>: ") + text_html)
for bridge in BridgesConfig:
if data.room.id in bridge:
if data.room.id not in bridge:
continue
rooms = list(bridge)
rooms.remove(data.room.id)
for room in rooms:
tokens = room.split(':')
SendMessage(SafeNamespace(platform=tokens[0]), ObjectUnion(data, {"room_id": ':'.join(tokens)}))
for room_id in rooms:
method(
SafeNamespace(platform=room_id.split(':')[0]),
ObjectUnion(data, {"room_id": room_id}, ({"text_plain": text_plain, "text_markdown": None, "text_html": text_html} if data.user else None)),
from_sent)
def UpdateUserDb(user:SafeNamespace) -> None:
def update_user_db(user:SafeNamespace) -> None:
if not (user and user.id):
return
try:
User.get(User.id == user.id)
except User.DoesNotExist:
@ -222,17 +243,17 @@ def UpdateUserDb(user:SafeNamespace) -> None:
except User.DoesNotExist:
User.create(id=user.id, id_hash=user_hash)
def DumpMessage(data:InputMessageData) -> None:
def dump_message(data:InputMessageData, prefix:str='') -> None:
if not (Debug and (DumpToFile or DumpToConsole)):
return
text = (data.text_plain.replace('\n', '\\n') if data.text_plain else '')
text = f"[{int(time.time())}] [{time.ctime()}] [{data.room and data.room.id}] [{data.message_id}] [{data.user.id}] {text}"
text = f"{prefix} [{int(time.time())}] [{time.ctime()}] [{data.room and data.room.id}] [{data.message_id}] [{data.user and data.user.id}] {text}"
if DumpToConsole:
print(text, data)
if DumpToFile:
open((DumpToFile if (DumpToFile and type(DumpToFile) == str) else "./Dump.txt"), 'a').write(text + '\n')
def SendMessage(context:EventContext, data:OutputMessageData) -> None:
def SendMessage(context:EventContext, data:OutputMessageData, from_sent:bool=False) -> None:
data = (OutputMessageData(**data) if type(data) == dict else data)
# TODO remove this after all modules are changed
@ -240,14 +261,16 @@ def SendMessage(context:EventContext, data:OutputMessageData) -> None:
data.text = data.Text
if data.TextPlain and not data.text_plain:
data.text_plain = data.TextPlain
if data.TextMarkdown and not data.text_markdown:
data.text_markdown = data.TextMarkdown
if data.text and not data.text_plain:
data.text_plain = data.text
if data.text_plain or data.text_markdown or data.text_html:
if data.text_html and not data.text_plain:
data.text_plain = BeautifulSoup(data.text_html, "html.parser").get_text()
elif data.text_markdown and not data.text_plain:
data.text_plain = data.text_markdown
elif data.text_plain and not data.text_html:
data.text_html = html_escape(data.text_plain)
elif data.text:
# our old system attempts to always receive Markdown and retransform when needed
data.text_plain = MdToTxt(data.text)
@ -266,7 +289,10 @@ def SendMessage(context:EventContext, data:OutputMessageData) -> None:
platform = Platforms[context.platform]
if (not context.manager) and (manager := platform.manager_class):
context.manager = (manager() if callable(manager) else manager)
return platform.sender(context, data)
result = platform.sender(context, data)
if not from_sent:
OnMessageSent(data)
return result
def SendNotice(context:EventContext, data) -> None:
pass
@ -275,7 +301,7 @@ def DeleteMessage(context:EventContext, data) -> None:
pass
def RegisterPlatform(name:str, main:callable, sender:callable, linker:callable=None, *, event_class=None, manager_class=None) -> None:
Platforms[name.lower()] = SafeNamespace(main=main, sender=sender, linker=linker, event_class=event_class, manager_class=manager_class)
Platforms[name.lower()] = SafeNamespace(name=name, main=main, sender=sender, linker=linker, event_class=event_class, manager_class=manager_class)
Log(f"{name}, ", inline=True)
def RegisterModule(name:str, endpoints:dict, *, group:str|None=None) -> None:
@ -317,9 +343,9 @@ def Main() -> None:
#SetupDb()
SetupLocales()
Log(f"📨️ Initializing Platforms... ", newline=False)
for platform in Platforms:
if Platforms[platform].main():
Log(f"{platform}, ", inline=True)
for platform in Platforms.values():
if platform.main():
Log(f"{platform.name}, ", inline=True)
Log("...Done. ✅️", inline=True, newline=True)
Log("🐶️ WinDog Ready!")
while True: