Code improvements; Add media descriptions in the HTML body

This commit is contained in:
octospacc 2022-10-29 11:18:52 +02:00
parent 368aec2225
commit c321613c89
3 changed files with 45 additions and 20 deletions

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
*.pyc *.pyc
*.db *.db
Config.py Config.py
Database.txt

View File

@ -20,8 +20,8 @@ MailPort = 465
LoopTime = 300 LoopTime = 300
# Additional time (in seconds) to sleep in code sections, to prevent ratelimiting. # Additional time (in seconds) to sleep in code sections, to prevent ratelimiting.
PageSleep = 2 # Between every scraped page PageSleep = 3 # Between every scraped page
ItemSleep = 1 # Between every scaped item ItemSleep = 2 # Between every scaped item
MailSleep = 9 # Between every sent mail MailSleep = 9 # Between every sent mail
# Stop recursive navigation across posts pages if limit is reached. Set 0 for no limit (use with caution on new profiles with many posts). # Stop recursive navigation across posts pages if limit is reached. Set 0 for no limit (use with caution on new profiles with many posts).
@ -30,6 +30,12 @@ MaxPagesRecursion = 10
# Whether or not to allow spaces in file names. # Whether or not to allow spaces in file names.
SpacesInFiles = True SpacesInFiles = True
### NOT IMPLEMENTED ###
# Whether or not to censor potentially private information in the logs, like email addresses (passwords are never printed despite of this setting).
#CensorLogs = True
### --------------- ###
# Other advanced settings. # Other advanced settings.
AppName = "MastodonFeedHTML" AppName = "bottocto-MastodonFeedHTML"
DbFile = "Database.txt"
UserAgent = AppName UserAgent = AppName

View File

@ -11,6 +11,12 @@ from email.mime.text import MIMEText
from urllib.request import urlopen, Request from urllib.request import urlopen, Request
from Config import * from Config import *
MediaDescsBlock = '<br><details><summary>Media descriptions</summary><ul>\n{Content}</ul></details>'
MainDivStyle = "word-wrap:break-word;"
AttachStyle = "max-width:100%; max-height:100vh;"
AvatarStyle = "max-height:4em;"
EmojiStyle = "max-height:1em;"
def SureList(Item): def SureList(Item):
return Item if type(Item) == list else [Item] return Item if type(Item) == list else [Item]
@ -25,6 +31,9 @@ def SleepPrint(s):
print(f"[I] Sleeping for {s}s...") print(f"[I] Sleeping for {s}s...")
time.sleep(s) time.sleep(s)
def MakeMediaDescsBlock(Content):
return MediaDescsBlock.format(Content=Content) if Content else ''
def HandleFeedsList(List): def HandleFeedsList(List):
for Feed in List: for Feed in List:
print(f"[I] Handling Feed ->\n: {Feed}") print(f"[I] Handling Feed ->\n: {Feed}")
@ -67,7 +76,7 @@ def HandleURL(IsFirstRun, URL, Usertag, IncludeRetoots, IncludeReplies, LocalSav
Index = 0 Index = 0
for Entry in Feed: for Entry in Feed:
Attached = '' MediaDescs, HTMLAttach, MailAttach = '', '', []
Anchor = Entry.find('a', class_='u-url') Anchor = Entry.find('a', class_='u-url')
if Anchor: if Anchor:
GlobalId = Anchor['href'].removeprefix('https://').removeprefix('http://') GlobalId = Anchor['href'].removeprefix('https://').removeprefix('http://')
@ -79,8 +88,8 @@ def HandleURL(IsFirstRun, URL, Usertag, IncludeRetoots, IncludeReplies, LocalSav
PageOlder = Anchor['href'] PageOlder = Anchor['href']
continue continue
if os.path.isfile(f'{AppName}.db'): if os.path.isfile(DbFile):
with open(f'{AppName}.db', 'r') as Db: with open(DbFile, 'r') as Db:
if f'{Usertag} {GlobalId}' in Db.read().splitlines(): if f'{Usertag} {GlobalId}' in Db.read().splitlines():
continue continue
@ -102,30 +111,25 @@ def HandleURL(IsFirstRun, URL, Usertag, IncludeRetoots, IncludeReplies, LocalSav
Title = Content.get_text().strip() Title = Content.get_text().strip()
Title = f"{Usertag}{StatusPrepend}: {Title[:32]}..." Title = f"{Usertag}{StatusPrepend}: {Title[:32]}..."
for Emoji in Entry.find_all('img', class_='custom-emoji'): # Custom emojis in text for Emoji in Entry.find_all('img', class_='custom-emoji'): # Custom emojis in text
Emoji['style'] = 'max-height:1em;' Emoji['style'] = EmojiStyle
Entry.find('img', class_='u-photo account__avatar')['style'] = 'max-height:4em;' # Profile pics Entry.find('img', class_='u-photo account__avatar')['style'] = AvatarStyle # Profile pics
Entry.find('div', class_='status__action-bar').replace_with('') Entry.find('div', class_='status__action-bar').replace_with('')
print(f"-> Item: {LocalId} - {Title}") print(f"-> Item: {LocalId} - {Title}")
HTML = f"""\ HTML = f"""\
<h1>{Title}</h1> <h1>{Title}</h1>
<div class="AppName-content" style="word-wrap:break-word;"> <div class="{AppName}-content" style="{MainDivStyle}">
{Entry} {Entry}
{{ Replace:Attached }} {{ Replace:MastodonFeedHTML:HTMLAttach }}
{{ Replace:MastodonFeedHTML:MediaDescs }}
</div> </div>
<br><hr><br> <br><hr><br>
<p><i>Via <a href="https://gitlab.com/octospacc/bottocto/-/tree/main/MastodonFeedHTML">https://gitlab.com/octospacc/bottocto/-/tree/main/MastodonFeedHTML</a></i></p> <p><i>Via <a href="https://gitlab.com/octospacc/bottocto/-/tree/main/MastodonFeedHTML">https://gitlab.com/octospacc/bottocto/-/tree/main/MastodonFeedHTML</a></i></p>
""" """
if SendMail:
Message = MIMEMultipart()
Message['From'] = MailUsername
Message['To'] = ', '.join(MailTo)
Message['Subject'] = Title
Message.attach(MIMEText(HTML.replace('{ Replace:Attached }', ''), 'html'))
Attachments = Entry.find('ul', class_='attachment-list__list') Attachments = Entry.find('ul', class_='attachment-list__list')
if Attachments and (LocalSave or SendMail): if Attachments and (LocalSave or SendMail):
@ -141,11 +145,13 @@ def HandleURL(IsFirstRun, URL, Usertag, IncludeRetoots, IncludeReplies, LocalSav
Response = urlopen(Request(Href, headers={'User-Agent':UserAgent})) Response = urlopen(Request(Href, headers={'User-Agent':UserAgent}))
Data = Response.read() Data = Response.read()
Mime = Response.info().get_content_type() Mime = Response.info().get_content_type()
if Alt:
MediaDescs += f'<li>{Alt}</li>\n'
if LocalSave: if LocalSave:
Tag = 'img' if Mime.split('/')[0] == 'image' else Mime.split('/')[0] Tag = 'img' if Mime.split('/')[0] == 'image' else Mime.split('/')[0]
Opening = f'<{Tag} alt="{Alt}" title="{Alt}"' if Tag == 'img' else f'<{Tag} controls' Opening = f'<{Tag} alt="{Alt}" title="{Alt}"' if Tag == 'img' else f'<{Tag} controls'
Closing = '>' if Tag == 'img' else f"></{Tag}>" Closing = '>' if Tag == 'img' else f"></{Tag}>"
Attached += f"""{Opening} style="max-width:100%; max-height:100vh;" src="data:{Mime};base64,{base64.b64encode(Data).decode()}"{Closing}\n""" HTMLAttach += f'<br>{Opening} style="{AttachStyle}" src="data:{Mime};base64,{base64.b64encode(Data).decode()}"{Closing}<br>\n'
if SendMail: if SendMail:
File = MIMEBase(Mime.split('/')[0], Mime.split('/')[1]) File = MIMEBase(Mime.split('/')[0], Mime.split('/')[1])
File.set_payload(Data) File.set_payload(Data)
@ -153,22 +159,34 @@ def HandleURL(IsFirstRun, URL, Usertag, IncludeRetoots, IncludeReplies, LocalSav
File.add_header( File.add_header(
"Content-Disposition", "Content-Disposition",
f"attachment; filename={Href.split('/')[-1]}") f"attachment; filename={Href.split('/')[-1]}")
Message.attach(File) MailAttach += [File]
if SendMail: if SendMail:
Message = MIMEMultipart()
Message['From'] = MailUsername
Message['To'] = ', '.join(MailTo)
Message['Subject'] = Title
Message.attach(MIMEText(HTML
.replace('{ Replace:MastodonFeedHTML:HTMLAttach }', '')
.replace('{ Replace:MastodonFeedHTML:MediaDescs }', MakeMediaDescsBlock(MediaDescs)), 'html'))
for File in MailAttach:
Message.attach(File)
with smtplib.SMTP_SSL(MailServer, MailPort, context=ssl.create_default_context()) as Client: with smtplib.SMTP_SSL(MailServer, MailPort, context=ssl.create_default_context()) as Client:
Client.login(MailUsername, MailPassword) Client.login(MailUsername, MailPassword)
Client.sendmail(MailUsername, MailTo, Message.as_string()) Client.sendmail(MailUsername, MailTo, Message.as_string())
SleepPrint(MailSleep) SleepPrint(MailSleep)
if LocalSave: if LocalSave:
LocalBackupDir = MakePathStr(Usertag) LocalBackupDir = MakePathStr(Usertag)
if not os.path.isdir(LocalBackupDir): if not os.path.isdir(LocalBackupDir):
os.mkdir(LocalBackupDir) os.mkdir(LocalBackupDir)
FileName = MakePathStr(f"{GlobalId.split('/')[-1]} - {Title}") FileName = MakePathStr(f"{GlobalId.split('/')[-1]} - {Title}")
with open(f'{LocalBackupDir}/{FileName}.html', 'w') as File: with open(f'{LocalBackupDir}/{FileName}.html', 'w') as File:
File.write(HTML.replace('{ Replace:Attached }', Attached)) File.write(HTML
.replace('{ Replace:MastodonFeedHTML:HTMLAttach }', HTMLAttach)
.replace('{ Replace:MastodonFeedHTML:MediaDescs }', MakeMediaDescsBlock(MediaDescs)))
with open(f'{AppName}.db', 'a') as Db: with open(DbFile, 'a') as Db:
Db.write(f'{Usertag} {GlobalId}' + '\n') Db.write(f'{Usertag} {GlobalId}' + '\n')
SleepPrint(ItemSleep) SleepPrint(ItemSleep)