More work on Matrix, move commands to new HTML locales, fix Mastodon

LibWinDog/Platforms/Mastodon/Mastodon.py

@@ -22,29 +22,42 @@ def MastodonMain() -> bool:
 	Mastodon = mastodon.Mastodon(api_base_url=MastodonUrl, access_token=MastodonToken)
 	class MastodonListener(mastodon.StreamListener):
 		def on_notification(self, event):
-			MastodonHandler(event)
+			MastodonHandler(event, Mastodon)
 	Mastodon.stream_user(MastodonListener(), run_async=True)
 	return True
 
-def MastodonHandler(event):
-	if event['type'] == 'mention':
-		#OnMessageParsed()
-		message = BeautifulSoup(event['status']['content'], 'html.parser').get_text(' ').strip().replace('\t', ' ')
-		if not message.split('@')[0]:
-			message = ' '.join('@'.join(message.split('@')[1:]).strip().split(' ')[1:]).strip()
-		if message[0] in CmdPrefixes:
-			command = ParseCmd(message)
-			if command:
-				command.messageId = event['status']['id']
-				if command.Name in Endpoints:
-					CallEndpoint(command.Name, EventContext(platform="mastodon", event=event, manager=Mastodon), command)
+def MastodonMakeInputMessageData(status:dict) -> InputMessageData:
+	data = InputMessageData(
+		message_id = ("mastodon:" + strip_url_scheme(status["uri"])),
+		text_html = status["content"],
+	)
+	data.text_plain = BeautifulSoup(data.text_html, "html.parser").get_text()
+	data.text_auto = GetWeightedText(data.text_html, data.text_plain)
+	command_tokens = data.text_plain.strip().replace("\t", " ").split(" ")
+	while command_tokens[0].strip().startswith('@') or not command_tokens[0]:
+		command_tokens.pop(0)
+	data.command = ParseCommand(" ".join(command_tokens))
+	data.user = SafeNamespace(
+		id = ("mastodon:" + strip_url_scheme(status["account"]["uri"])),
+		name = status["account"]["display_name"],
+	)
+	data.user.settings = (GetUserSettings(data.user.id) or SafeNamespace())
+	return data
+
+def MastodonHandler(event, Mastodon):
+	if event["type"] == "mention":
+		data = MastodonMakeInputMessageData(event["status"])
+		OnMessageParsed(data)
+		if (command := ObjGet(data, "command.name")):
+			CallEndpoint(command, EventContext(platform="mastodon", event=event, manager=Mastodon), data)
 
 def MastodonSender(context:EventContext, data:OutputMessageData, destination) -> None:
 	media_results = None
 	if data.media:
 		media_results = []
-		for medium in data.media[:4]: # Mastodon limits posts to 4 attachments
-			medium_result = context.manager.media_post(medium, Magic(mime=True).from_buffer(medium))
+		# TODO support media by url (do we have to upload them or can just pass the original URL?)
+		for medium in data.media[:4]: # Mastodon limits posts to 4 attachments, so we drop any more
+			medium_result = context.manager.media_post(medium["bytes"], Magic(mime=True).from_buffer(medium["bytes"]))
 			while medium_result["url"] == "null":
 				medium_result = context.manager.media(medium_result)
 			media_results.append(medium_result)

LibWinDog/Platforms/Matrix/Matrix.py

@@ -7,30 +7,64 @@
 
 # MatrixUrl = "https://matrix.example.com"
 # MatrixUsername = "username"
+
+# Provide either your password, or an active access_token below.
 # MatrixPassword = "hunter2"
 
+# If logging in via password, a token will be automatically generated and saved to Config.
+# MatrixToken = ""
+
 # end windog config # """
 
-MatrixUrl, MatrixUsername, MatrixPassword = None, None, None
+MatrixUrl, MatrixUsername, MatrixPassword, MatrixToken = None, None, None, None
 
+from asyncio import run as asyncio_run
 import nio
-import simplematrixbotlib as MatrixBotLib
-from threading import Thread
+#from nio import AsyncClient, MatrixRoom, RoomMessageText
+#import simplematrixbotlib as MatrixBotLib
+
+async def MatrixMessageHandler(room:nio.MatrixRoom, event:nio.RoomMessage) -> None:
+	data = MatrixMakeInputMessageData(room, event)
+
+def MatrixMakeInputMessageData(room:nio.MatrixRoom, event:nio.RoomMessage) -> InputMessageData:
+	data = InputMessageData(
+		message_id = f"matrix:{event.event_id}",
+		room = SafeNamespace(
+			id = f"matrix:{room.room_id}",
+			name = room.display_name,
+		),
+		user = SafeNamespace(
+			id = f"matrix:{event.sender}",
+		)
+	)
+	print(data)
+	return data
 
 def MatrixMain() -> bool:
-	if not (MatrixUrl and MatrixUsername and MatrixPassword):
+	if not (MatrixUrl and MatrixUsername and (MatrixPassword or MatrixToken)):
 		return False
-	MatrixBot = MatrixBotLib.Bot(MatrixBotLib.Creds(MatrixUrl, MatrixUsername, MatrixPassword))
-	@MatrixBot.listener.on_message_event
-	@MatrixBot.listener.on_custom_event(nio.events.room_events.RoomMessageFile)
-	async def MatrixMessageListener(room, message) -> None:
-		pass
-		#print(message)
-		#match = MatrixBotLib.MessageMatch(room, message, MatrixBot)
-		#OnMessageParsed()
-		#if match.is_not_from_this_bot() and match.command("windogtest"):
-		#	pass #await MatrixBot.api.send_text_message(room.room_id, " ".join(arg for arg in match.args()))
-	Thread(target=lambda:MatrixBot.run()).start()
+	#MatrixBot = MatrixBotLib.Bot(MatrixBotLib.Creds(MatrixUrl, MatrixUsername, MatrixPassword))
+	##@MatrixBot.listener.on_message_event
+	#@MatrixBot.listener.on_custom_event(nio.RoomMessageText)
+	#async def MatrixMessageListener(room, message, event) -> None:
+	#	print(message)
+	#	#match = MatrixBotLib.MessageMatch(room, message, MatrixBot)
+	#	#OnMessageParsed()
+	#	#if match.is_not_from_this_bot() and match.command("windogtest"):
+	#	#	pass #await MatrixBot.api.send_text_message(room.room_id, " ".join(arg for arg in match.args()))
+	#@MatrixBot.listener.on_custom_event(nio.RoomMessageFile)
+	#async def MatrixMessageFileListener(room, event):
+	#	print(event)
+	#Thread(target=lambda:MatrixBot.run()).start()
+	async def client() -> None:
+		client = nio.AsyncClient(MatrixUrl, MatrixUsername)
+		login = await client.login(password=MatrixPassword, token=MatrixToken)
+		if MatrixPassword and (not MatrixToken) and (token := ObjGet(login, "access_token")):
+			open("./Config.py", 'a').write(f'\n# Added automatically #\nMatrixToken = "{token}"\n')
+		await client.sync(30000) # resync old messages first to "skip read ones"
+		client.add_event_callback(MatrixMessageHandler, nio.RoomMessage)
+		await client.sync_forever(timeout=30000)
+	Thread(target=lambda:asyncio_run(client())).start()
 	return True
 
 def MatrixSender() -> None:

LibWinDog/Platforms/Matrix/requirements.txt

@@ -1 +1,2 @@
+matrix-nio
 simplematrixbotlib

LibWinDog/Platforms/Telegram/Telegram.py

@@ -23,7 +23,7 @@ def TelegramMain() -> bool:
 		return False
 	updater = telegram.ext.Updater(TelegramToken)
 	dispatcher = updater.dispatcher
-	dispatcher.add_handler(MessageHandler(Filters.text | Filters.command, TelegramHandlerWrapper))
+	dispatcher.add_handler(MessageHandler(Filters.text | Filters.command, TelegramHandler))
 	updater.start_polling()
 	#app = Application.builder().token(TelegramToken).build()
 	#app.add_handler(MessageHandler(filters.TEXT | filters.COMMAND, TelegramHandler))
@@ -40,60 +40,33 @@ def TelegramMakeInputMessageData(message:telegram.Message) -> InputMessageData:
 	)
 	data.text_auto = GetWeightedText(data.text_markdown, data.text_plain)
 	data.command = ParseCommand(data.text_plain)
-	data.room = SafeNamespace(
-		id = f"telegram:{message.chat.id}",
-		tag = message.chat.username,
-		name = (message.chat.title or message.chat.first_name),
-	)
 	data.user = SafeNamespace(
 		id = f"telegram:{message.from_user.id}",
 		tag = message.from_user.username,
 		name = message.from_user.first_name,
 	)
-	data.user.settings = GetUserSettings(data.user.id)
+	data.user.settings = (GetUserSettings(data.user.id) or SafeNamespace())
+	data.room = SafeNamespace(
+		id = f"telegram:{message.chat.id}",
+		tag = message.chat.username,
+		name = (message.chat.title or message.chat.first_name),
+	)
+	linked = TelegramLinker(data)
+	data.message_url = linked.message
+	data.room.url = linked.room
 	return data
 
-def TelegramHandlerWrapper(update:telegram.Update, context:CallbackContext=None) -> None:
-	Thread(target=lambda:TelegramHandlerCore(update, context)).start()
-
-def TelegramHandlerCore(update:telegram.Update, context:CallbackContext=None) -> None:
+def TelegramHandler(update:telegram.Update, context:CallbackContext=None) -> None:
+	def handler() -> None:
 		if not update.message:
 			return
 		data = TelegramMakeInputMessageData(update.message)
-	if update.message.reply_to_message:
-		data.quoted = TelegramMakeInputMessageData(update.message.reply_to_message)
+		if (quoted := update.message.reply_to_message):
+			data.quoted = TelegramMakeInputMessageData(quoted)
 		OnMessageParsed(data)
-	cmd = ParseCmd(update.message.text)
-	if cmd:
-		# TODO remove old cmd and just pass the data object
-		cmd.command = data.command
-		cmd.quoted = data.quoted
-		cmd.user = data.user
-		cmd.message_id = data.message_id
-		cmd.messageId = update.message.message_id
-		cmd.TextPlain = cmd.Body
-		cmd.TextMarkdown = update.message.text_markdown_v2
-		cmd.Text = GetWeightedText(cmd.TextMarkdown, cmd.TextPlain)
-		if cmd.Tokens[0][0] in CmdPrefixes and cmd.Name in Endpoints:
-			cmd.User = SimpleNamespace(**{
-				"Name": update.message.from_user.first_name,
-				"Tag": update.message.from_user.username,
-				"Id": f'telegram:{update.message.from_user.id}',
-			})
-			if update.message.reply_to_message:
-				cmd.Quoted = SimpleNamespace(**{
-					"messageId": update.message.reply_to_message.message_id,
-					"Body": update.message.reply_to_message.text,
-					"TextPlain": update.message.reply_to_message.text,
-					"TextMarkdown": update.message.reply_to_message.text_markdown_v2,
-					"Text": GetWeightedText(update.message.reply_to_message.text_markdown_v2, update.message.reply_to_message.text),
-					"User": SimpleNamespace(**{
-						"Name": update.message.reply_to_message.from_user.first_name,
-						"Tag": update.message.reply_to_message.from_user.username,
-						"Id": f'telegram:{update.message.reply_to_message.from_user.id}',
-					}),
-				})
-			CallEndpoint(cmd.Name, EventContext(platform="telegram", event=update, manager=context), cmd)
+		if (command := ObjGet(data, "command.name")):
+			CallEndpoint(command, EventContext(platform="telegram", event=update, manager=context), data)
+	Thread(target=handler).start()
 
 def TelegramSender(context:EventContext, data:OutputMessageData, destination):
 	result = None
@@ -104,7 +77,7 @@ def TelegramSender(context:EventContext, data:OutputMessageData, destination):
 		if data.media:
 			for medium in data.media:
 				result = context.event.message.reply_photo(
-					(DictGet(medium, "bytes") or DictGet(medium, "url")),
+					(ObjGet(medium, "bytes") or ObjGet(medium, "url")),
 					caption=(data.text_html or data.text_markdown or data.text_plain),
 					parse_mode=("HTML" if data.text_html else "MarkdownV2" if data.text_markdown else None),
 					reply_to_message_id=replyToId)
@@ -116,14 +89,17 @@ def TelegramSender(context:EventContext, data:OutputMessageData, destination):
 			result = context.event.message.reply_text(data.text_plain, reply_to_message_id=replyToId)
 	return TelegramMakeInputMessageData(result)
 
+# TODO support usernames
 def TelegramLinker(data:InputMessageData) -> SafeNamespace:
 	linked = SafeNamespace()
 	if data.room.id:
-		room_id = data.room.id.split("telegram:")[1]
-		linked.room = f"https://t.me/{room_id}"
+		# prefix must be dropped for groups and channels, while direct chats apparently can never be linked
+		if (room_id := "100".join(data.room.id.split("telegram:")[1].split("100")[1:])):
+			# apparently Telegram doesn't really support links to rooms by id without a message id, so we just use a null one
+			linked.room = f"https://t.me/c/{room_id}/0"
 			if data.message_id:
 				message_id = data.message_id.split("telegram:")[1]
-		linked.message = f"{linked.room}/{message_id}"
+				linked.message = f"https://t.me/c/{room_id}/{message_id}"
 	return linked
 
 RegisterPlatform(name="Telegram", main=TelegramMain, sender=TelegramSender, linker=TelegramLinker, eventClass=telegram.Update)

Locale/en.json

@@ -1,15 +1,7 @@
 {
-	"start": [
-		"*Hi* {0}*!*\n\nUse /help to read a list of available commands."
-	],
 	"help": [
 		"*There's no one around to help (yet).*"
 	],
-	"echo": {
-		"empty": [
-			"*Echo what? Give me something to repeat.*"
-		]
-	},
 	"wish": {
 		"empty": [
 			"*You wished for nothing! ✨*\n\n_Nothing happens..._"

Locale/it.json

@@ -1,15 +1,7 @@
 {
-	"start": [
-		"*Ciao* {0}*!*\n\nUsa /help per leggere la lista dei comandi."
-	],
 	"help": [
 		"*Non c'è nessuno qui ad aiutarti (per ora).*"
 	],
-	"echo": {
-		"empty": [
-			"*Echo cosa? Dimmi qualcosa da ripetere.*"
-		]
-	},
 	"wish": {
 		"empty": [
 			"*Non hai desiderato nulla! ✨*\n\n_Non succede niente..._"

ModWinDog/Base/Base.py

@@ -3,11 +3,6 @@
 #  Licensed under AGPLv3 by OctoSpacc  #
 # ==================================== #
 
-import re, subprocess
-
-def cStart(context:EventContext, data:InputMessageData) -> None:
-	SendMessage(context, {"Text": choice(Locale.__('start')).format(data.User.Name)})
-
 def cSource(context:EventContext, data:InputMessageData) -> None:
 	SendMessage(context, {"TextPlain": ("""\
 * Original Code: {https://gitlab.com/octospacc/WinDog}
@@ -20,8 +15,12 @@ def cGdpr(context:EventContext, data:InputMessageData) -> None:
 def cConfig(context:EventContext, data:InputMessageData) -> None:
 	if not (settings := GetUserSettings(data.user.id)):
 		User.update(settings=EntitySettings.create()).where(User.id == data.user.id).execute()
-	if (get := ObjGet(data, "command.arguments.get")):
-		SendMessage(context, OutputMessageData(text_plain=str(ObjGet(data.user.settings, get))))
+	if (to_set := ObjGet(data, "command.arguments.set")):
+		pass # TODO set in db, but first we need to ensure data is handled safely
+	if (to_get := ObjGet(data, "command.arguments.get")):
+		# TODO show a hint on possible options?
+		return SendMessage(context, OutputMessageData(text_plain=str(ObjGet(data.user.settings, to_get))))
+	# TODO show general help when no useful parameters are passed
 	#Cmd = TelegramHandleCmd(update)
 	#if not Cmd: return
 	# ... area: eu, us, ...
@@ -39,30 +38,14 @@ def cPing(context:EventContext, data:InputMessageData) -> None:
 def cEval(context:EventContext, data:InputMessageData) -> None:
 	SendMessage(context, {"Text": choice(Locale.__('eval'))})
 
-def cExec(context:EventContext, data:InputMessageData) -> None:
-	if len(data.Tokens) >= 2 and data.Tokens[1].lower() in ExecAllowed:
-		cmd = data.Tokens[1].lower()
-		Out = subprocess.run(('sh', '-c', f'export PATH=$PATH:/usr/games; {cmd}'),
-			stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode()
-		# <https://stackoverflow.com/a/14693789>
-		Caption = (re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])').sub('', Out))
-		SendMessage(context, {
-			"TextPlain": Caption,
-			"TextMarkdown": MarkdownCode(Caption, True),
-		})
-	else:
-		SendMessage(context, {"Text": choice(Locale.__('eval'))})
-
-RegisterModule(name="Misc", endpoints=[
-	SafeNamespace(names=["start"], summary="Salutes the user, hinting that the bot is working and providing basic quick help.", handler=cStart),
+RegisterModule(name="Base", endpoints=[
 	SafeNamespace(names=["source"], summary="Provides a copy of the bot source codes and/or instructions on how to get it.", handler=cSource),
 	SafeNamespace(names=["config"], handler=cConfig, arguments={
 		"get": True,
 	}),
 	#SafeNamespace(names=["gdpr"], summary="Operations for european citizens regarding your personal data.", handler=cGdpr),
 	SafeNamespace(names=["ping"], summary="Responds pong, useful for testing messaging latency.", handler=cPing),
-	SafeNamespace(names=["eval"], summary="Execute a Python command (or safe literal operation) in the current context. Currently not implemented.", handler=cEval),
-	SafeNamespace(names=["exec"], summary="Execute a system command from the allowed ones and return stdout+stderr.", handler=cExec),
+	#SafeNamespace(names=["eval"], summary="Execute a Python command (or safe literal operation) in the current context. Currently not implemented.", handler=cEval),
 	#SafeNamespace(names=["format"], summary="Reformat text using an handful of rules. Not yet implemented.", handler=cFormat),
 	#SafeNamespace(names=["frame"], summary="Frame someone's message into a platform-styled image. Not yet implemented.", handler=cFrame),
 	#SafeNamespace(names=["repeat"], summary="I had this planned but I don't remember what this should have done. Not yet implemented.", handler=cRepeat),

ModWinDog/Echo/Echo.py

@@ -4,22 +4,20 @@
 # ==================================== #
 
 def cEcho(context:EventContext, data:InputMessageData) -> None:
-	text = ObjGet(data, "command.body")
-	if text:
-		prefix = "🗣️ "
+	if not (text := ObjGet(data, "command.body")):
+		return SendMessage(context, OutputMessageData(text_html=context.endpoint.get_string("empty", data.user.settings.language)))
+	prefix = f'<a href="{data.message_url}">🗣️</a> '
 	#prefix = f"[🗣️]({context.linker(data).message}) "
-		if len(data.Tokens) == 2:
+	if len(data.command.tokens) == 2:
 		nonascii = True
-			for char in data.Tokens[1]:
+		for char in data.command.tokens[1]:
 			if ord(char) < 256:
 				nonascii = False
 				break
 		if nonascii:
 			# text is not ascii, probably an emoji (altough not necessarily), so just pass as is (useful for Telegram emojis)
 			prefix = ''
-		SendMessage(context, OutputMessageData(text=(prefix + text)))
-	else:
-		SendMessage(context, OutputMessageData(text_html=context.endpoint.get_string('empty')))
+	SendMessage(context, OutputMessageData(text_html=(prefix + html_escape(text))))
 
 RegisterModule(name="Echo", endpoints=[
 	SafeNamespace(names=["echo"], handler=cEcho),

ModWinDog/Echo/Echo.yaml

@@ -1,3 +1,5 @@
+summary:
+	en: Tools for repeating messages.
 endpoints:
 	echo:
 		summary:

ModWinDog/GPT/GPT.py

@@ -8,12 +8,12 @@ from g4f.client import Client as G4FClient
 g4fClient = G4FClient()
 
 def cGpt(context:EventContext, data:InputMessageData) -> None:
-	if not data.command.body:
+	if not (prompt := data.command.body):
 		return SendMessage(context, {"Text": "You must type some text."})
 	output = None
 	while not output or output.startswith("sorry, 您的ip已由于触发防滥用检测而被封禁,本服务网址是"): # quick fix for a strange ratelimit message
 		output = ""
-		for completion in g4fClient.chat.completions.create(model="gpt-3.5-turbo", messages=[{"role": "user", "content": data.command.body}], stream=True):
+		for completion in g4fClient.chat.completions.create(model="gpt-3.5-turbo", messages=[{"role": "user", "content": prompt}], stream=True):
 			output += (completion.choices[0].delta.content or "")
 	return SendMessage(context, {"TextPlain": f"[🤖️ GPT]\n\n{output}"})
 

ModWinDog/Hashing/Hashing.py

@@ -11,13 +11,10 @@ def cHash(context:EventContext, data:InputMessageData):
 	if not (text_input and (algorithm in hashlib.algorithms_available)):
 		return SendMessage(context, {"Text": choice(Locale.__('hash.usage')).format(data.command.tokens[0], hashlib.algorithms_available)})
 	hashed = hashlib.new(algorithm, text_input.encode()).hexdigest()
-	return SendMessage(context, {
-		"TextPlain": hashed,
-		"TextMarkdown": MarkdownCode(hashed, True),
-	})
+	return SendMessage(context, OutputMessageData(text_plain=hashed, text_html=f"<pre>{hashed}</pre>"))
 
-RegisterModule(name="Hashing", group="Geek", summary="Functions for hashing of textual content.", endpoints=[
-	SafeNamespace(names=["hash"], summary="Responds with the hash-sum of a message received.", handler=cHash, arguments={
+RegisterModule(name="Hashing", group="Geek", endpoints=[
+	SafeNamespace(names=["hash"], handler=cHash, arguments={
 		"algorithm": True,
 	}),
 ])

ModWinDog/Hashing/Hashing.yaml

@@ -1,7 +1,8 @@
 summary:
 	en: Functions for calculating hashes of content.
 	it: Funzioni per calcolare hash di contenuti.
-hash:
+endpoints:
+	hash:
 		summary:
 			en: Responds with the hash-sum of the received message.
 		arguments:

ModWinDog/Help/Help.py

@@ -5,17 +5,18 @@
 
 # TODO: implement /help <commandname> feature
 def cHelp(context:EventContext, data:InputMessageData) -> None:
-	moduleList = ''
+	module_list = ''
+	language = data.user.settings.language
 	for module in Modules:
-		summary = Modules[module].summary
+		summary = Modules[module].get_string("summary", language)#summary
 		endpoints = Modules[module].endpoints
-		moduleList += (f"\n\n{module}" + (f": {summary}" if summary else ''))
+		module_list += (f"\n\n{module}" + (f": {summary}" if summary else ''))
 		for endpoint in endpoints:
-			summary = endpoint.summary
-			moduleList += (f"\n* /{', /'.join(endpoint.names)}" + (f": {summary}" if summary else ''))
-	SendMessage(context, {"Text": f"[ Available Modules ]{moduleList}"})
+			summary = Modules[module].get_string(f"endpoints.{endpoint.names[0]}.summary", language)
+			module_list += (f"\n* /{', /'.join(endpoint.names)}" + (f": {summary}" if summary else ''))
+	SendMessage(context, OutputMessageData(text=module_list))
 
 RegisterModule(name="Help", group="Basic", endpoints=[
-	SafeNamespace(names=["help"], summary="Provides help for the bot. For now, it just lists the commands.", handler=cHelp),
+	SafeNamespace(names=["help"], handler=cHelp),
 ])
 

ModWinDog/Help/Help.yaml

@@ -0,0 +1,5 @@
+endpoints:
+	help:
+		summary:
+			en: Provides help for the bot. For now, it just lists the commands.
+

ModWinDog/Internet/Internet.py

@@ -10,19 +10,18 @@ MicrosoftBingSettings = {}
 """ # end windog config # """
 
 from urlextract import URLExtract
-from urllib import parse as UrlParse
 from urllib.request import urlopen, Request
 
 def HttpReq(url:str, method:str|None=None, *, body:bytes=None, headers:dict[str, str]={"User-Agent": WebUserAgent}):
 	return urlopen(Request(url, method=method, data=body, headers=headers))
 
 def cEmbedded(context:EventContext, data:InputMessageData) -> None:
-	if len(data.Tokens) >= 2:
+	if len(data.command.tokens) >= 2:
 		# Find links in command body
-		Text = (data.TextMarkdown + ' ' + data.TextPlain)
-	elif data.Quoted and data.Quoted.Text:
+		Text = (data.text_markdown + ' ' + data.text_plain)
+	elif data.quoted and data.quoted.text_auto:
 		# Find links in quoted message
-		Text = (data.Quoted.TextMarkdown + ' ' + data.Quoted.TextPlain)
+		Text = (data.quoted.text_markdown + ' ' + data.quoted.text_plain)
 	else:
 		# TODO Error message
 		return
@@ -41,38 +40,37 @@ def cEmbedded(context:EventContext, data:InputMessageData) -> None:
 			url = "https://hlb0.octt.eu.org/cors-main.php/https://" + url
 			proto = ''
 		else:
-			if urlDomain == "instagram.com":
+			if urlDomain in ("instagram.com", "www.instagram.com"):
 				urlDomain = "ddinstagram.com"
 			elif urlDomain in ("twitter.com", "x.com"):
 				urlDomain = "fxtwitter.com"
 			elif urlDomain == "vm.tiktok.com":
 				urlDomain = "vm.vxtiktok.com"
-			url = urlDomain + url[len(urlDomain):]
+			url = (urlDomain + '/' + '/'.join(url.split('/')[1:]))
 		SendMessage(context, {"TextPlain": f"{{{proto}{url}}}"})
 	# else TODO error message?
 
 def cWeb(context:EventContext, data:InputMessageData) -> None:
-	if data.Body:
+	if not (query := data.command.body):
+		return # TODO show message
 	try:
-			QueryUrl = UrlParse.quote(data.Body)
+		QueryUrl = urlparse.quote(query)
 		Req = HttpReq(f'https://html.duckduckgo.com/html?q={QueryUrl}')
-			Caption = f'🦆🔎 "{data.Body}": https://duckduckgo.com/?q={QueryUrl}\n\n'
+		Caption = f'🦆🔎 "{query}": https://duckduckgo.com/?q={QueryUrl}\n\n'
 		Index = 0
 		for Line in Req.read().decode().replace('\t', ' ').splitlines():
 			if ' class="result__a" ' in Line and ' href="//duckduckgo.com/l/?uddg=' in Line:
 				Index += 1
-					Link = UrlParse.unquote(Line.split(' href="//duckduckgo.com/l/?uddg=')[1].split('&rut=')[0])
+				Link = urlparse.unquote(Line.split(' href="//duckduckgo.com/l/?uddg=')[1].split('&rut=')[0])
 				Title = Line.strip().split('</a>')[0].strip().split('</span>')[-1].strip().split('>')
 				if len(Title) > 1:
-						Title = HtmlUnescape(Title[1].strip())
+					Title = html_unescape(Title[1].strip())
 					Caption += f'[{Index}] {Title} : {{{Link}}}\n\n'
 				else:
 					continue
 		SendMessage(context, {"TextPlain": f'{Caption}...'})
 	except Exception:
 		raise
-	else:
-		pass
 
 def cImages(context:EventContext, data:InputMessageData) -> None:
 	pass
@@ -82,57 +80,57 @@ def cNews(context:EventContext, data:InputMessageData) -> None:
 
 def cTranslate(context:EventContext, data:InputMessageData) -> None:
 	language_to = data.command.arguments["language_to"]
-	text_input = (data.command.body or (data.Quoted and data.Quoted.Body))
+	text_input = (data.command.body or (data.quoted and data.quoted.text_plain))
 	if not (text_input and language_to):
 		return SendMessage(context, {"TextPlain": f"Usage: /translate <to language> <text>"})
 	try:
 		# TODO: Use many different public Lingva instances in rotation to avoid overloading a specific one
-		result = json.loads(HttpReq(f'https://lingva.ml/api/v1/auto/{language_to}/{UrlParse.quote(text_input)}').read())
+		result = json.loads(HttpReq(f'https://lingva.ml/api/v1/auto/{language_to}/{urlparse.quote(text_input)}').read())
 		SendMessage(context, {"TextPlain": f"[{result['info']['detectedSource']} (auto) -> {language_to}]\n\n{result['translation']}"})
 	except Exception:
 		raise
 
 # unsplash source appears to be deprecated! <https://old.reddit.com/r/unsplash/comments/s13x4h/what_happened_to_sourceunsplashcom/l65epl8/>
-def cUnsplash(context:EventContext, data:InputMessageData) -> None:
-	try:
-		Req = HttpReq(f'https://source.unsplash.com/random/?{UrlParse.quote(data.Body)}')
-		ImgUrl = Req.geturl().split('?')[0]
-		SendMessage(context, {
-			"TextPlain": f'{{{ImgUrl}}}',
-			"TextMarkdown": MarkdownCode(ImgUrl, True),
-			"Media": Req.read(),
-		})
-	except Exception:
-		raise
+#def cUnsplash(context:EventContext, data:InputMessageData) -> None:
+#	try:
+#		Req = HttpReq(f'https://source.unsplash.com/random/?{urlparse.quote(data.command.body)}')
+#		ImgUrl = Req.geturl().split('?')[0]
+#		SendMessage(context, {
+#			"TextPlain": f'{{{ImgUrl}}}',
+#			"TextMarkdown": MarkdownCode(ImgUrl, True),
+#			"Media": Req.read(),
+#		})
+#	except Exception:
+#		raise
 
 def cSafebooru(context:EventContext, data:InputMessageData) -> None:
 	ApiUrl = 'https://safebooru.org/index.php?page=dapi&s=post&q=index&limit=100&tags='
 	try:
-		if data.Body:
+		img_id, img_url = None, None
+		if (query := data.command.body):
 			for i in range(7): # retry a bunch of times if we can't find a really random result
-				ImgUrls = HttpReq(f'{ApiUrl}md5:{RandHexStr(3)}%20{UrlParse.quote(data.Body)}').read().decode().split(' file_url="')[1:]
+				ImgUrls = HttpReq(f'{ApiUrl}md5:{RandHexStr(3)}%20{urlparse.quote(query)}').read().decode().split(' file_url="')[1:]
 				if ImgUrls:
 					break
 			if not ImgUrls: # literal search
-				ImgUrls = HttpReq(f'{ApiUrl}{UrlParse.quote(data.Body)}').read().decode().split(' file_url="')[1:]
+				ImgUrls = HttpReq(f'{ApiUrl}{urlparse.quote(query)}').read().decode().split(' file_url="')[1:]
 			if not ImgUrls:
 				return SendMessage(context, {"Text": "Error: Could not get any result from Safebooru."})
 			ImgXml = choice(ImgUrls)
-			ImgUrl = ImgXml.split('"')[0]
-			ImgId = ImgXml.split(' id="')[1].split('"')[0]
+			img_url = ImgXml.split('"')[0]
+			img_id = ImgXml.split(' id="')[1].split('"')[0]
 		else:
 			HtmlReq = HttpReq(HttpReq('https://safebooru.org/index.php?page=post&s=random').geturl())
 			for Line in HtmlReq.read().decode().replace('\t', ' ').splitlines():
 				if '<img ' in Line and ' id="image" ' in Line and ' src="':
-					ImgUrl = Line.split(' src="')[1].split('"')[0]
-					ImgId = ImgUrl.split('?')[-1]
+					img_url = Line.split(' src="')[1].split('"')[0]
+					img_id = img_url.split('?')[-1]
 					break
-		if ImgUrl:
-			SendMessage(context, {
-				"TextPlain": f'[{ImgId}]\n{{{ImgUrl}}}',
-				"TextMarkdown": (f'\\[`{ImgId}`\\]\n' + MarkdownCode(ImgUrl, True)),
-				"media": {"url": ImgUrl}, #, "bytes": HttpReq(ImgUrl).read()},
-			})
+		if img_url:
+			SendMessage(context, OutputMessageData(
+				text_plain=f"[{img_id}]\n{{{img_url}}}",
+				text_html=f"[<code>{img_id}</code>]\n<pre>{img_url}</pre>",
+				media={"url": img_url}))
 		else:
 			pass
 	except Exception as error:

ModWinDog/Multifun/Multifun.py

@@ -4,19 +4,19 @@
 # ==================================== #
 
 def mMultifun(context:EventContext, data:InputMessageData) -> None:
-	cmdkey = data.Name
+	cmdkey = data.command.name
 	replyToId = None
-	if data.Quoted:
-		replyFromUid = data.Quoted.User.Id
+	if data.quoted:
+		replyFromUid = data.quoted.user.id
 		# TODO work on all platforms for the bot id
 		if replyFromUid.split(':')[1] == TelegramToken.split(':')[0] and 'bot' in Locale.__(cmdkey):
 			Text = choice(Locale.__(f'{cmdkey}.bot'))
-		elif replyFromUid == data.User.Id and 'self' in Locale.__(cmdkey):
-			Text = choice(Locale.__(f'{cmdkey}.self')).format(data.User.Name)
+		elif replyFromUid == data.user.id and 'self' in Locale.__(cmdkey):
+			Text = choice(Locale.__(f'{cmdkey}.self')).format(data.user.name)
 		else:
 			if 'others' in Locale.__(cmdkey):
-				Text = choice(Locale.__(f'{cmdkey}.others')).format(data.User.Name, data.Quoted.User.Name)
-				replyToId = data.Quoted.messageId
+				Text = choice(Locale.__(f'{cmdkey}.others')).format(data.user.name, data.quoted.user.name)
+				replyToId = data.quoted.message_id
 	else:
 		if 'empty' in Locale.__(cmdkey):
 			Text = choice(Locale.__(f'{cmdkey}.empty'))

ModWinDog/Percenter/Percenter.py

@@ -4,8 +4,8 @@
 # ==================================== #
 
 def mPercenter(context:EventContext, data:InputMessageData) -> None:
-	SendMessage(context, {"Text": choice(Locale.__(f'{data.Name}.{"done" if data.Body else "empty"}')).format(
-		Cmd=data.Tokens[0], Percent=RandPercent(), Thing=data.Body)})
+	SendMessage(context, {"Text": choice(Locale.__(f'{data.command.name}.{"done" if data.command.body else "empty"}')).format(
+		Cmd=data.command.tokens[0], Percent=RandPercent(), Thing=data.command.body)})
 
 RegisterModule(name="Percenter", endpoints=[
 	SafeNamespace(names=["wish", "level"], summary="Provides fun trough percentage-based toys.", handler=mPercenter),

ModWinDog/Scrapers/Scrapers.py

@@ -36,8 +36,7 @@ def closeSelenium(index:int, driver:Driver) -> None:
 
 def cDalleSelenium(context:EventContext, data:InputMessageData) -> None:
 	warning_text = "has been blocked by Microsoft because it violates their content policy. Further attempts might lead to a ban on your profile. Please review the Code of Conduct for Image Creator in this picture or at https://www.bing.com/new/termsofuseimagecreator#content-policy."
-	prompt = data.command.body
-	if not prompt:
+	if not (prompt := data.command.body):
 		return SendMessage(context, {"Text": "Please tell me what to generate."})
 	driver_index, driver = None, None
 	try:
@@ -74,11 +73,10 @@ def cDalleSelenium(context:EventContext, data:InputMessageData) -> None:
 				img_url = img_url.get_attribute("src").split('?')[0]
 				img_array.append({"url": img_url}) #, "bytes": HttpReq(img_url).read()})
 			page_url = driver.current_url.split('?')[0]
-			SendMessage(context, {
-				"TextPlain": f'"{prompt}"\n{{{page_url}}}',
-				"TextMarkdown": (f'"_{CharEscape(prompt, "MARKDOWN")}_"\n' + MarkdownCode(page_url, True)),
-				"media": img_array,
-			})
+			SendMessage(context, OutputMessageData(
+				text_plain=f'"{prompt}"\n{{{page_url}}}',
+				text_html=f'"<i>{html_escape(prompt)}</i>"\n<pre>{page_url}</pre>',
+				media=img_array))
 			return closeSelenium(driver_index, driver)
 		raise Exception("VM timed out.")
 	except Exception as error:
@@ -87,8 +85,7 @@ def cDalleSelenium(context:EventContext, data:InputMessageData) -> None:
 		closeSelenium(driver_index, driver)
 
 def cCraiyonSelenium(context:EventContext, data:InputMessageData) -> None:
-	prompt = data.command.body
-	if not prompt:
+	if not (prompt := data.command.body):
 		return SendMessage(context, {"Text": "Please tell me what to generate."})
 	driver_index, driver = None, None
 	try:

ModWinDog/Scripting/Scripting.py

@@ -25,7 +25,7 @@ def luaAttributeFilter(obj, attr_name, is_setting):
 # TODO make print behave the same as normal Lua, and expose a function for printing without newlines
 def cLua(context:EventContext, data:InputMessageData) -> None:
 	# TODO update quoted api getting
-	scriptText = (data.command.body or (data.Quoted and data.Quoted.Body))
+	scriptText = (data.command.body or (data.quoted and data.quoted.text_plain))
 	if not scriptText:
 		return SendMessage(context, {"Text": "You must provide some Lua code to execute."})
 	luaRuntime = NewLuaRuntime(max_memory=LuaMemoryLimit, register_eval=False, register_builtins=False, attribute_filter=luaAttributeFilter)

ModWinDog/Start/Start.py

@@ -0,0 +1,14 @@
+# ==================================== #
+#  WinDog multi-purpose chatbot        #
+#  Licensed under AGPLv3 by OctoSpacc  #
+# ==================================== #
+
+def cStart(context:EventContext, data:InputMessageData) -> None:
+	SendMessage(context, OutputMessageData(
+		text_html=context.endpoint.get_string(
+			"start", data.user.settings.language).format(data.user.name)))
+
+RegisterModule(name="Start", endpoints=[
+	SafeNamespace(names=["start"], handler=cStart),
+])
+

ModWinDog/Start/Start.yaml

@@ -0,0 +1,16 @@
+summary:
+	en: Things related to starting the bot, on supported platforms.
+endpoints:
+	start:
+		summary:
+			en: Salutes the user, hinting that the bot is working and providing basic quick help.
+		start:
+			en: |
+				<b>Hi</b> {0}<b>!</b>
+				
+				Use /help to read a list of available commands.
+			it: |
+				<b>Ciao</b> {0}<b>!</b>
+				
+				Usa /help per leggere la lista dei comandi.
+

ModWinDog/System/System.py

@@ -0,0 +1,24 @@
+# ==================================== #
+#  WinDog multi-purpose chatbot        #
+#  Licensed under AGPLv3 by OctoSpacc  #
+# ==================================== #
+
+import subprocess
+from re import compile as re_compile
+
+def cExec(context:EventContext, data:InputMessageData) -> None:
+	if not (len(data.command.tokens) >= 2 and data.command.tokens[1].lower() in ExecAllowed):
+		return SendMessage(context, {"Text": choice(Locale.__('eval'))})
+	command = data.command.tokens[1].lower()
+	output = subprocess.run(
+		("sh", "-c", f"export PATH=$PATH:/usr/games; {command}"),
+		stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode()
+	# <https://stackoverflow.com/a/14693789>
+	text = (re_compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])").sub('', output))
+	SendMessage(context, OutputMessageData(
+		text_plain=text, text_html=f"<pre>{html_escape(text)}</pre>"))
+
+RegisterModule(name="System", endpoints=[
+	SafeNamespace(names=["exec"], handler=cExec),
+])
+

ModWinDog/System/System.yaml

@@ -0,0 +1,5 @@
+endpoints:
+	exec:
+		summary:
+			en: Execute a system command from the allowed ones and return stdout+stderr.
+

WinDog.py

@@ -8,13 +8,15 @@ import json, time
 from binascii import hexlify
 from glob import glob
 from hashlib import new as hashlib_new
+from html import escape as html_escape, unescape as html_unescape
 from os import listdir
 from os.path import isfile, isdir
-from random import choice, randint
+from random import choice, choice as randchoice, randint
+from threading import Thread
 from traceback import format_exc
+from urllib import parse as urlparse
 from yaml import load as yaml_load, BaseLoader as yaml_BaseLoader
 from bs4 import BeautifulSoup
-from html import unescape as HtmlUnescape
 from markdown import markdown
 from LibWinDog.Types import *
 from LibWinDog.Config import *
@@ -98,17 +100,21 @@ def ObjGet(node:object, query:str, /) -> any:
 			return None
 	return node
 
-def isinstanceSafe(clazz:any, instance:any) -> bool:
+def isinstanceSafe(clazz:any, instance:any, /) -> bool:
 	if instance != None:
 		return isinstance(clazz, instance)
 	return False
 
-def GetString(bank:dict, query:str|dict, lang:str=None):
+def get_string(bank:dict, query:str|dict, lang:str=None, /):
 	if not (result := ObjGet(bank, f"{query}.{lang or DefaultLang}")):
 		if not (result := ObjGet(bank, f"{query}.en")):
-			result = tuple(ObjGet(bank, query).values())[0]
+			result = ObjGet(bank, query)
 	return result
 
+def strip_url_scheme(url:str) -> str:
+	tokens = urlparse.urlparse(url)
+	return f"{tokens.netloc}{tokens.path}"
+
 def CharEscape(String:str, Escape:str='') -> str:
 	if Escape == 'MARKDOWN':
 		return escape_markdown(String, version=2)
@@ -128,9 +134,6 @@ def InferMdEscape(raw:str, plain:str) -> str:
 			chars += char
 	return chars
 
-def MarkdownCode(text:str, block:bool) -> str:
-	return ('```\n' + text.strip().replace('`', '\`') + '\n```')
-
 def MdToTxt(md:str) -> str:
 	return BeautifulSoup(markdown(md), 'html.parser').get_text(' ')
 
@@ -141,23 +144,6 @@ def HtmlEscapeFull(Raw:str) -> str:
 		New += f'&#x{Hex[i] + Hex[i+1]};'
 	return New
 
-def GetRawTokens(text:str) -> list:
-	return text.strip().replace('\t', ' ').replace('  ', ' ').replace('  ', ' ').split(' ')
-
-def ParseCmd(msg) -> SafeNamespace|None:
-	#if not len(msg) or msg[1] not in CmdPrefixes:
-	#	return
-	name = msg.replace('\n', ' ').replace('\t', ' ').replace('  ', ' ').replace('  ', ' ').split(' ')[0][1:].split('@')[0]
-	#if not name:
-	#	return
-	return SafeNamespace(**{
-		"Name": name.lower(),
-		"Body": name.join(msg.split(name)[1:]).strip(),
-		"Tokens": GetRawTokens(msg),
-		"User": None,
-		"Quoted": None,
-	})
-
 def GetWeightedText(*texts) -> str|None:
 	for text in texts:
 		if text:
@@ -179,6 +165,7 @@ def GetUserSettings(user_id:str) -> SafeNamespace|None:
 	except EntitySettings.DoesNotExist:
 		return None
 
+# TODO handle @ characters attached to command, e.g. on telegram
 def ParseCommand(text:str) -> SafeNamespace|None:
 	if not text:
 		return None
@@ -227,8 +214,8 @@ def UpdateUserDb(user:SafeNamespace) -> None:
 def DumpMessage(data:InputMessageData) -> None:
 	if not (Debug and (DumpToFile or DumpToConsole)):
 		return
-	text = (data.text_auto.replace('\n', '\\n') if data.text_auto else '')
-	text = f"[{int(time.time())}] [{time.ctime()}] [{data.room.id}] [{data.message_id}] [{data.user.id}] {text}"
+	text = (data.text_plain.replace('\n', '\\n') if data.text_auto else '')
+	text = f"[{int(time.time())}] [{time.ctime()}] [{data.room and data.room.id}] [{data.message_id}] [{data.user.id}] {text}"
 	if DumpToConsole:
 		print(text, data)
 	if DumpToFile:
@@ -247,13 +234,13 @@ def SendMessage(context:EventContext, data:OutputMessageData, destination=None)
 
 	if data.text_plain or data.text_markdown or data.text_html:
 		if data.text_html and not data.text_plain:
-			data.text_plain = data.text_html # TODO flatten the HTML to plaintext
+			data.text_plain = BeautifulSoup(data.text_html, "html.parser").get_text()
 		elif data.text_markdown and not data.text_plain:
 			data.text_plain = data.text_markdown
 	elif data.text:
 		# our old system attempts to always receive Markdown and retransform when needed
 		data.text_plain = MdToTxt(data.text)
-		data.text_markdown = CharEscape(HtmlUnescape(data.text), InferMdEscape(HtmlUnescape(data.text), data.text_plain))
+		data.text_markdown = CharEscape(html_unescape(data.text), InferMdEscape(html_unescape(data.text), data.text_plain))
 		#data.text_html = ???
 	if data.media:
 		data.media = SureArray(data.media)
@@ -272,10 +259,10 @@ def RegisterPlatform(name:str, main:callable, sender:callable, linker:callable=N
 	Log(f"{name}, ", inline=True)
 
 def RegisterModule(name:str, endpoints:dict, *, group:str|None=None, summary:str|None=None) -> None:
-	module = SafeNamespace(group=group, endpoints=endpoints)
+	module = SafeNamespace(group=group, endpoints=endpoints, get_string=(lambda query, lang=None, /: None))
 	if isfile(file := f"./ModWinDog/{name}/{name}.yaml"):
 		module.strings = yaml_load(open(file, 'r').read().replace("\t", "    "), Loader=yaml_BaseLoader)
-		module.get_string = (lambda query, lang=None: GetString(module.strings, query, lang))
+		module.get_string = (lambda query, lang=None: get_string(module.strings, query, lang))
 	Modules[name] = module
 	Log(f"{name}, ", inline=True)
 	for endpoint in endpoints:
@@ -287,7 +274,7 @@ def CallEndpoint(name:str, context:EventContext, data:InputMessageData):
 	endpoint = Endpoints[name]
 	context.endpoint = endpoint
 	context.module = endpoint.module
-	context.endpoint.get_string = (lambda query, lang=None: endpoint.module.get_string(f"endpoints.{data.command.name}.{query}", lang))
+	context.endpoint.get_string = (lambda query, lang=None, /: endpoint.module.get_string(f"endpoints.{data.command.name}.{query}", lang))
 	return endpoint.handler(context, data)
 
 def WriteNewConfig() -> None: