diff --git a/.gitignore b/.gitignore index 5f4b2dc..e9dde15 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.pyc *.db Config.py +Database.txt diff --git a/MastodonFeedHTML/Example.Config.py b/MastodonFeedHTML/Example.Config.py index 722d9ab..9076d3b 100644 --- a/MastodonFeedHTML/Example.Config.py +++ b/MastodonFeedHTML/Example.Config.py @@ -20,8 +20,8 @@ MailPort = 465 LoopTime = 300 # Additional time (in seconds) to sleep in code sections, to prevent ratelimiting. -PageSleep = 2 # Between every scraped page -ItemSleep = 1 # Between every scaped item +PageSleep = 3 # Between every scraped page +ItemSleep = 2 # Between every scaped item MailSleep = 9 # Between every sent mail # Stop recursive navigation across posts pages if limit is reached. Set 0 for no limit (use with caution on new profiles with many posts). @@ -30,6 +30,12 @@ MaxPagesRecursion = 10 # Whether or not to allow spaces in file names. SpacesInFiles = True +### NOT IMPLEMENTED ### +# Whether or not to censor potentially private information in the logs, like email addresses (passwords are never printed despite of this setting). +#CensorLogs = True +### --------------- ### + # Other advanced settings. -AppName = "MastodonFeedHTML" +AppName = "bottocto-MastodonFeedHTML" +DbFile = "Database.txt" UserAgent = AppName diff --git a/MastodonFeedHTML/MastodonFeedHTML.py b/MastodonFeedHTML/MastodonFeedHTML.py index afe8c2e..abeb9a8 100755 --- a/MastodonFeedHTML/MastodonFeedHTML.py +++ b/MastodonFeedHTML/MastodonFeedHTML.py @@ -11,6 +11,12 @@ from email.mime.text import MIMEText from urllib.request import urlopen, Request from Config import * +MediaDescsBlock = '
Media descriptions
' +MainDivStyle = "word-wrap:break-word;" +AttachStyle = "max-width:100%; max-height:100vh;" +AvatarStyle = "max-height:4em;" +EmojiStyle = "max-height:1em;" + def SureList(Item): return Item if type(Item) == list else [Item] @@ -25,6 +31,9 @@ def SleepPrint(s): print(f"[I] Sleeping for {s}s...") time.sleep(s) +def MakeMediaDescsBlock(Content): + return MediaDescsBlock.format(Content=Content) if Content else '' + def HandleFeedsList(List): for Feed in List: print(f"[I] Handling Feed ->\n: {Feed}") @@ -67,7 +76,7 @@ def HandleURL(IsFirstRun, URL, Usertag, IncludeRetoots, IncludeReplies, LocalSav Index = 0 for Entry in Feed: - Attached = '' + MediaDescs, HTMLAttach, MailAttach = '', '', [] Anchor = Entry.find('a', class_='u-url') if Anchor: GlobalId = Anchor['href'].removeprefix('https://').removeprefix('http://') @@ -79,8 +88,8 @@ def HandleURL(IsFirstRun, URL, Usertag, IncludeRetoots, IncludeReplies, LocalSav PageOlder = Anchor['href'] continue - if os.path.isfile(f'{AppName}.db'): - with open(f'{AppName}.db', 'r') as Db: + if os.path.isfile(DbFile): + with open(DbFile, 'r') as Db: if f'{Usertag} {GlobalId}' in Db.read().splitlines(): continue @@ -102,30 +111,25 @@ def HandleURL(IsFirstRun, URL, Usertag, IncludeRetoots, IncludeReplies, LocalSav Title = Content.get_text().strip() Title = f"{Usertag}{StatusPrepend}: {Title[:32]}..." for Emoji in Entry.find_all('img', class_='custom-emoji'): # Custom emojis in text - Emoji['style'] = 'max-height:1em;' - Entry.find('img', class_='u-photo account__avatar')['style'] = 'max-height:4em;' # Profile pics + Emoji['style'] = EmojiStyle + Entry.find('img', class_='u-photo account__avatar')['style'] = AvatarStyle # Profile pics Entry.find('div', class_='status__action-bar').replace_with('') print(f"-> Item: {LocalId} - {Title}") HTML = f"""\

{Title}

-
+
{Entry} -{{ Replace:Attached }} +{{ Replace:MastodonFeedHTML:HTMLAttach }} +{{ Replace:MastodonFeedHTML:MediaDescs }}


Via https://gitlab.com/octospacc/bottocto/-/tree/main/MastodonFeedHTML

""" - if SendMail: - Message = MIMEMultipart() - Message['From'] = MailUsername - Message['To'] = ', '.join(MailTo) - Message['Subject'] = Title - Message.attach(MIMEText(HTML.replace('{ Replace:Attached }', ''), 'html')) Attachments = Entry.find('ul', class_='attachment-list__list') if Attachments and (LocalSave or SendMail): @@ -141,11 +145,13 @@ def HandleURL(IsFirstRun, URL, Usertag, IncludeRetoots, IncludeReplies, LocalSav Response = urlopen(Request(Href, headers={'User-Agent':UserAgent})) Data = Response.read() Mime = Response.info().get_content_type() + if Alt: + MediaDescs += f'
  • {Alt}
    • \n' if LocalSave: Tag = 'img' if Mime.split('/')[0] == 'image' else Mime.split('/')[0] Opening = f'<{Tag} alt="{Alt}" title="{Alt}"' if Tag == 'img' else f'<{Tag} controls' Closing = '>' if Tag == 'img' else f">" - Attached += f"""{Opening} style="max-width:100%; max-height:100vh;" src="data:{Mime};base64,{base64.b64encode(Data).decode()}"{Closing}\n""" + HTMLAttach += f'
    {Opening} style="{AttachStyle}" src="data:{Mime};base64,{base64.b64encode(Data).decode()}"{Closing}
    \n' if SendMail: File = MIMEBase(Mime.split('/')[0], Mime.split('/')[1]) File.set_payload(Data) @@ -153,22 +159,34 @@ def HandleURL(IsFirstRun, URL, Usertag, IncludeRetoots, IncludeReplies, LocalSav File.add_header( "Content-Disposition", f"attachment; filename={Href.split('/')[-1]}") - Message.attach(File) + MailAttach += [File] if SendMail: + Message = MIMEMultipart() + Message['From'] = MailUsername + Message['To'] = ', '.join(MailTo) + Message['Subject'] = Title + Message.attach(MIMEText(HTML + .replace('{ Replace:MastodonFeedHTML:HTMLAttach }', '') + .replace('{ Replace:MastodonFeedHTML:MediaDescs }', MakeMediaDescsBlock(MediaDescs)), 'html')) + for File in MailAttach: + Message.attach(File) with smtplib.SMTP_SSL(MailServer, MailPort, context=ssl.create_default_context()) as Client: Client.login(MailUsername, MailPassword) Client.sendmail(MailUsername, MailTo, Message.as_string()) SleepPrint(MailSleep) + if LocalSave: LocalBackupDir = MakePathStr(Usertag) if not os.path.isdir(LocalBackupDir): os.mkdir(LocalBackupDir) FileName = MakePathStr(f"{GlobalId.split('/')[-1]} - {Title}") with open(f'{LocalBackupDir}/{FileName}.html', 'w') as File: - File.write(HTML.replace('{ Replace:Attached }', Attached)) + File.write(HTML + .replace('{ Replace:MastodonFeedHTML:HTMLAttach }', HTMLAttach) + .replace('{ Replace:MastodonFeedHTML:MediaDescs }', MakeMediaDescsBlock(MediaDescs))) - with open(f'{AppName}.db', 'a') as Db: + with open(DbFile, 'a') as Db: Db.write(f'{Usertag} {GlobalId}' + '\n') SleepPrint(ItemSleep)