diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 62eb323..b8d4d34 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ image: alpine:latest before_script: | apk update - apk add python3 + apk add python3 make pages: stage: deploy diff --git a/App/Source/Build.py b/App/Source/Build.py index b8ce501..323d4d6 100644 --- a/App/Source/Build.py +++ b/App/Source/Build.py @@ -15,6 +15,7 @@ import time from ast import literal_eval from datetime import datetime from pathlib import Path +from Modules.Assets import * from Modules.Config import * from Modules.Gemini import * from Modules.Globals import * @@ -22,15 +23,8 @@ from Modules.Logging import * from Modules.Markdown import * from Modules.Site import * from Modules.Sitemap import * +from Modules.Social import * from Modules.Utils import * -try: - from Modules.ActivityPub import * - ActivityPub = True -except: - logging.warning("⚠ Can't load the ActivityPub module. Its use is disabled. Make sure the 'requests' library is installed.") - ActivityPub = False -from Libs import rcssmin -cssmin = rcssmin._make_cssmin(python_only=True) def ResetOutDir(OutDir): for e in (OutDir, f'{OutDir}.Content', f'{OutDir}.gmi'): @@ -120,7 +114,7 @@ def WriteRedirects(Flags, Pages, FinalPaths, Locale): StrRedirect=Locale['IfNotRedirected'])) def BuildMain(Args, FeedEntries): - Flags, Snippets, FinalPaths = {}, {}, [] + Flags, Snippets = {}, {} HavePages, HavePosts = False, False SiteConf = LoadConfFile('Site.ini') @@ -232,34 +226,17 @@ def BuildMain(Args, FeedEntries): Locale=Locale, Threads=Threads) + # REFACTOR: The functions below are still not changed to accept a Page as Dict + for i, e in enumerate(Pages): + Pages[i] = list(e.values()) + if FeedEntries != 0: logging.info("Generating Feeds") for FeedType in (True, False): MakeFeed(Flags, Pages, FeedType) - if ActivityPub and MastodonURL and MastodonToken and SiteDomain: - logging.info("Mastodon Stuff") - MastodonPosts = MastodonShare(Flags, Pages, Locale) - else: - MastodonPosts = [] - - for File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image in Pages: - if IsLightRun(File, LimitFiles): - continue - File = f"{OutDir}/{StripExt(File)}.html" - Content = ReadFile(File) - Post = '' - for p in MastodonPosts: - if p['Link'] == SiteDomain + '/' + File[len(f"{OutDir}/"):]: - Post = HTMLCommentsBlock.format( - StrComments=Locale['Comments'], - StrOpen=Locale['OpenInNewTab'], - URL=p['Post']) - break - Content = ReplWithEsc(Content, '[staticoso:Comments]', Post) - Content = ReplWithEsc(Content, '', Post) - WriteFile(File, Content) - FinalPaths += [File] + logging.info("Applying Social Integrations") + FinalPaths = ApplySocialIntegrations(Flags, Pages, LimitFiles, Locale) logging.info("Creating Redirects") WriteRedirects(Flags, Pages, FinalPaths, Locale) @@ -279,20 +256,7 @@ def BuildMain(Args, FeedEntries): MakeSitemap(Flags, Pages) logging.info("Preparing Assets") - #os.system(f"cp -R Assets/* {OutDir}/") - if Flags['MinifyAssets']: - shutil.copytree('Assets', OutDir, ignore=IgnoreFiles, dirs_exist_ok=True) - for File in Path('Assets').rglob('*'): - if os.path.isfile(File): - Dest = f"{OutDir}/{str(File)[len('Assets')+1:]}" - if str(File).lower().endswith(FileExtensions['HTML']): - WriteFile(Dest, DoMinifyHTML(ReadFile(File), MinifyKeepComments)) - elif str(File).lower().endswith('.css'): - WriteFile(Dest, cssmin(ReadFile(File), MinifyKeepComments)) - else: - shutil.copy2(File, Dest) - else: - shutil.copytree('Assets', OutDir, dirs_exist_ok=True) + PrepareAssets(Flags) if __name__ == '__main__': StartTime = time.time() diff --git a/App/Source/Modules/Assets.py b/App/Source/Modules/Assets.py new file mode 100644 index 0000000..be8a2ae --- /dev/null +++ b/App/Source/Modules/Assets.py @@ -0,0 +1,31 @@ +""" ================================== | +| This file is part of | +| staticoso | +| Just a simple Static Site Generator | +| | +| Licensed under the AGPLv3 license | +| Copyright (C) 2022-2023, OctoSpacc | +| ================================== """ + +import shutil +from Modules.HTML import DoMinifyHTML +from Modules.Utils import * + +from Libs import rcssmin +cssmin = rcssmin._make_cssmin(python_only=True) + +def PrepareAssets(Flags): + f = NameSpace(Flags) + if f.MinifyAssets: + shutil.copytree('Assets', f.OutDir, ignore=IgnoreFiles, dirs_exist_ok=True) + for File in Path('Assets').rglob('*'): + if os.path.isfile(File): + Dest = f"{f.OutDir}/{str(File)[len('Assets')+1:]}" + if str(File).lower().endswith(FileExtensions['HTML']): + WriteFile(Dest, DoMinifyHTML(ReadFile(File), f.MinifyKeepComments)) + elif str(File).lower().endswith('.css'): + WriteFile(Dest, cssmin(ReadFile(File), f.MinifyKeepComments)) + else: + shutil.copy2(File, Dest) + else: + shutil.copytree('Assets', f.OutDir, dirs_exist_ok=True) diff --git a/App/Source/Modules/Elements.py b/App/Source/Modules/Elements.py index cf3532f..2a55c04 100644 --- a/App/Source/Modules/Elements.py +++ b/App/Source/Modules/Elements.py @@ -156,7 +156,7 @@ def FormatTitles(Titles:list, Flatten=False): # Clean up a generic HTML tree such that it's compliant with the HTML Journal standard # (https://m15o.ichi.city/site/subscribing-to-a-journal-page.html); # basis is: find an element with the JournalBody attr., and group its direct children as
s -def MakeHTMLJournal(Flags, Locale, FilePath, HTML): +def MakeHTMLJournal(Flags:dict, Locale:dict, FilePath:str, HTML:str): Soup, Journal, Entries = MkSoup(HTML), '', [] for t in Soup.find_all(attrs={"htmljournal":True}): #JournalStyle = JournalStyles[t.attrs["journalstyle"]] if 'journalstyle' in t.attrs and t.attrs["journalstyle"] in JournalStyles else JournalStyles['Default'] diff --git a/App/Source/Modules/Feed.py b/App/Source/Modules/Feed.py index 5001dbb..1bfd133 100644 --- a/App/Source/Modules/Feed.py +++ b/App/Source/Modules/Feed.py @@ -12,7 +12,7 @@ from Libs.feedgen.feed import FeedGenerator from Modules.Utils import * -def MakeFeed(Flags, Pages, FullSite=False): +def MakeFeed(Flags:dict, Pages:list, FullSite=False): CategoryFilter = Flags['FeedCategoryFilter'] MaxEntries = Flags['FeedEntries'] diff --git a/App/Source/Modules/Gemini.py b/App/Source/Modules/Gemini.py index 30ab5c5..be691f3 100644 --- a/App/Source/Modules/Gemini.py +++ b/App/Source/Modules/Gemini.py @@ -13,7 +13,7 @@ from Libs.bs4 import BeautifulSoup from Modules.HTML import * from Modules.Utils import * -def FixGemlogDateLine(Line): +def FixGemlogDateLine(Line:str): if len(Line) >= 2 and Line[0] == '[' and Line[1].isdigit(): Line = Line[1:] else: @@ -22,7 +22,7 @@ def FixGemlogDateLine(Line): Line = Words[0] + '\n' + Words[1][1:] + ' ' + ' '.join(Words[2:]) return Line -def GemtextCompileList(Flags, Pages, LimitFiles): +def GemtextCompileList(Flags:dict, Pages:list, LimitFiles): Cmd = '' for File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image in Pages: if IsLightRun(File, LimitFiles): @@ -53,7 +53,7 @@ def FindEarliest(Str, Items): Str.find(Item) return Pos, Item -def ParseTag(Content): - print(Content) +def ParseTag(Content:str): + #print(Content) Parse = BeautifulSoup(str(Content), 'html.parser') Tag = Parse.find() diff --git a/App/Source/Modules/HTML.py b/App/Source/Modules/HTML.py index f85a4f0..a6a44a9 100644 --- a/App/Source/Modules/HTML.py +++ b/App/Source/Modules/HTML.py @@ -15,30 +15,31 @@ from Modules.Utils import * # Suppress useless bs4 warnings warnings.filterwarnings('ignore', message='The input looks more like a filename than markup.') +warnings.filterwarnings('ignore', message='The soupsieve package is not installed.') -def MkSoup(HTML): - return BeautifulSoup(HTML, 'html.parser') +def MkSoup(Html:str): + return BeautifulSoup(Html, 'html.parser') -def StripAttrs(HTML): - Soup = MkSoup(HTML) +def StripAttrs(Html:str): + Soup = MkSoup(Html) Tags = Soup.find_all() for t in Tags: if 'href' not in t.attrs and 'src' not in t.attrs: t.attrs = {} return str(Soup) -def StripTags(HTML, ToStrip): # Remove desired tags from the HTML - Soup = MkSoup(HTML) +def StripTags(Html:str, ToStrip:list): # Remove desired tags from the HTML + Soup = MkSoup(Html) Tags = Soup.find_all() for t in Tags: if t.name in ToStrip: t.replace_with('') return str(Soup) -def DoHTMLFixPre(HTML): - if not ("
" in HTML or "
" in Html or "
').removesuffix('

') + Title = markdown(MarkdownHTMLEscape(File, f.MarkdownExts), extensions=f.MarkdownExts).removeprefix('

').removesuffix('

') else: - Title = MakeListTitle(File, Meta, Titles, 'HTMLTitle', BlogName, PathPrefix) + Title = MakeListTitle(File, Meta, Titles, 'HTMLTitle', f.BlogName, PathPrefix) if SingleLine: List += ' ' + Title + ' ' else: @@ -122,7 +124,10 @@ def FindPreprocLine(Line:str, Meta, Macros): # IgnoreBlocksStart += [l] return (Meta, Macros, Changed) -def PagePreprocessor(Path:str, TempPath:str, Type:str, SiteTemplate, SiteRoot, GlobalMacros, CategoryUncategorized, LightRun:bool=False, Content=None): +def PagePreprocessor(Flags:dict, Page:list, SiteTemplate, GlobalMacros, LightRun:bool=False): + CategoryUncategorized = Flags['CategoriesUncategorized'] + Path, TempPath, Type, Content = Page + File = ReadFile(Path) if not Content else Content Path = Path.lower() Content, Titles, DashyTitles, HTMLTitlesFound, Macros, Meta, MetaDefault = '', [], [], False, '', '', { diff --git a/App/Source/Modules/Pug.py b/App/Source/Modules/Pug.py index 62fbf3a..6b8a168 100644 --- a/App/Source/Modules/Pug.py +++ b/App/Source/Modules/Pug.py @@ -12,12 +12,12 @@ import os from Modules.Utils import * -def PugCompileList(OutputDir, Pages, LimitFiles): +def PugCompileList(OutDir:str, Pages:list, LimitFiles): # Pug-cli seems to shit itself with folder paths as input, so we pass ALL the files as arguments Paths = '' for File, Content, Titles, Meta in Pages: if File.lower().endswith('.pug') and (LimitFiles == False or File in LimitFiles): - Path = f'{OutputDir}/{File}' + Path = f'{OutDir}/{File}' WriteFile(Path, Content) Paths += f'"{Path}" ' if Paths: diff --git a/App/Source/Modules/Site.py b/App/Source/Modules/Site.py index 8dbe30b..3451a64 100644 --- a/App/Source/Modules/Site.py +++ b/App/Source/Modules/Site.py @@ -20,7 +20,13 @@ from Modules.Meta import * from Modules.Pug import * from Modules.Utils import * -def PatchHTML(File, HTML, StaticPartsText, DynamicParts, DynamicPartsText, HTMLPagesList, PagePath, Content, Titles, Meta, SiteDomain, SiteRoot, SiteName, BlogName, FolderRoots, Categories, SiteLang, Locale, LightRun): +def PatchHTML(Flags, File, HTML, StaticPartsText, DynamicParts, DynamicPartsText, HTMLPagesList, PagePath, Content, Titles, Meta, FolderRoots, Categories, Locale, LightRun): + SiteDomain = Flags['SiteDomain'] + SiteRoot = Flags['SiteRoot'] + SiteLang = Flags['SiteLang'] + SiteName = Flags['SiteName'] + BlogName = Flags['BlogName'] + HTMLTitles = FormatTitles(Titles) BodyDescription, BodyImage = '', '' if not File.lower().endswith('.txt'): @@ -171,8 +177,9 @@ def PatchHTML(File, HTML, StaticPartsText, DynamicParts, DynamicPartsText, HTMLP return HTML, ContentHTML, Description, Image -def HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, Locale): +def HandlePage(Flags:dict, Page:list, Pages, Categories, LimitFiles, Snippets, ConfMenu, Locale:dict): File, Content, Titles, Meta = Page + OutDir, MarkdownExts, Sorting, MinifyKeepComments = Flags['OutDir'], Flags['MarkdownExts'], Flags['Sorting'], Flags['MinifyKeepComments'] SiteName, BlogName, SiteTagline = Flags['SiteName'], Flags['BlogName'], Flags['SiteTagline'] SiteTemplate, SiteLang = Flags['SiteTemplate'], Flags['SiteLang'] @@ -200,17 +207,16 @@ def HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, L else: TemplateMeta = TemplatePreprocessor(TemplatesText[Meta['Template']]) HTMLPagesList = GetHTMLPagesList( + Flags, Pages=Pages, - BlogName=BlogName, - SiteRoot=SiteRoot, PathPrefix=GetPathLevels(File), Unite=ConfMenu, Type='Page', For='Menu', - MarkdownExts=MarkdownExts, MenuStyle=TemplateMeta['MenuStyle']) HTML, ContentHTML, Description, Image = PatchHTML( + Flags, File=File, HTML=TemplatesText[Meta['Template']], StaticPartsText=StaticPartsText, @@ -221,26 +227,19 @@ def HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, L Content=Content, Titles=Titles, Meta=Meta, - SiteDomain=SiteDomain, - SiteRoot=SiteRoot, - SiteName=SiteName, - BlogName=BlogName, FolderRoots=FolderRoots, Categories=Categories, - SiteLang=SiteLang, Locale=Locale, LightRun=LightRun) HTML = ReplWithEsc(HTML, f"", GetHTMLPagesList( + Flags, Limit=Flags['FeedEntries'], Type='Post', Category=None if Flags['FeedCategoryFilter'] == '*' else Flags['FeedCategoryFilter'], Pages=Pages, - BlogName=BlogName, - SiteRoot=SiteRoot, PathPrefix=GetPathLevels(File), For='Categories', - MarkdownExts=MarkdownExts, MenuStyle='Flat', ShowPaths=False)) if 'staticoso:DirectoryList:' in HTML: # Reduce risk of unnecessary cycles @@ -249,14 +248,12 @@ def HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, L if Line.startswith(''): Path = Line[len('", DirectoryList) @@ -287,6 +284,7 @@ def HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, L if not LightRun and 'htmljournal' in ContentHTML.lower(): # Avoid extra cycles HTML, _, _, _ = PatchHTML( + Flags, File=File, HTML=TemplatesText[Meta['Template']], StaticPartsText=StaticPartsText, @@ -297,32 +295,92 @@ def HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, L Content=MakeHTMLJournal(Flags, Locale, f'{StripExt(File)}.html', ContentHTML), Titles='', Meta=Meta, - SiteDomain=SiteDomain, - SiteRoot=SiteRoot, - SiteName=SiteName, - BlogName=BlogName, FolderRoots=FolderRoots, Categories=Categories, - SiteLang=SiteLang, Locale=Locale, LightRun=LightRun) if Flags["JournalRedirect"]: HTML = HTML.replace('', f"""""") WriteFile(StripExt(PagePath)+'.Journal.html', HTML) - return [File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image] + #return [File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image] + return {"File":File, "Content":Content, "Titles":Titles, "Meta":Meta, "ContentHtml":ContentHTML, "SlimHtml":SlimHTML, "Description":Description, "Image":Image} -def MultiprocPagePreprocessor(d): - PrintProcPercentDots(d['Process'], 2) - return PagePreprocessor(d['Path'], d['TempPath'], d['Type'], d['Template'], d['SiteRoot'], d['GlobalMacros'], d['CategoryUncategorized'], d['LightRun']) +def MultiprocPagePreprocessor(d:dict): + PrintProcPercentDots(d['Process'])#, 2) + return PagePreprocessor(d['Flags'], d['Page'], d['Template'], d['GlobalMacros'], d['LightRun']) -def MultiprocHandlePage(d): +def MultiprocHandlePage(d:dict): PrintProcPercentDots(d['Process']) return HandlePage(d['Flags'], d['Page'], d['Pages'], d['Categories'], d['LimitFiles'], d['Snippets'], d['ConfMenu'], d['Locale']) -def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale, Threads): - PagesPaths, PostsPaths, Pages, MadePages, Categories = [], [], [], [], {} +def FindPagesPaths(): + Paths = {"Pages":[], "Posts":[]} + for Ext in FileExtensions['Pages']: + for Type in ('Pages', 'Posts'): + for File in Path(Type).rglob(f'*.{Ext}'): + Paths[Type] += [FileToStr(File, f'{Type}/')] + return Paths + +def ReorderPagesPaths(Paths:dict, Sorting:dict): + for Type in ('Pages', 'Posts'): + Paths[Type] = FileNameDateSort(Paths[Type]) + if Sorting[Type] in ('Inverse', 'Reverse'): + Paths[Type].reverse() + return Paths + +def PopulateCategoryLists(Flags:dict, Pages:list, Categories): + for Cat in Categories: + for Type in ('Page', 'Post'): + Categories[Cat] += GetHTMLPagesList( + Flags, + Pages=Pages, + PathPrefix=GetPathLevels('Categories/'), + Type=Type, + Category=Cat, + For='Categories', + MenuStyle='Flat') + return Categories + +def MakeAutoCategories(Flags:dict, Categories): + Pages = [] + if Flags['CategoriesAutomatic']: + OutDir = Flags['OutDir'] + Dir = f'{OutDir}/Categories' + for Cat in Categories: + Exists = False + for File in Path(Dir).rglob(str(Cat)+'.*'): + Exists = True + break + if not Exists: + File = f'Categories/{Cat}.md' + FilePath = f'{OutDir}/{File}' + WriteFile(FilePath, CategoryPageTemplate.format(Name=Cat)) + _, Content, Titles, Meta = PagePreprocessor(Flags, [FilePath, FilePath, Type, None], SiteTemplate, GlobalMacros, LightRun=LightRun) + Pages += [File, Content, Titles, Meta] + return Pages + +def PreprocessSourcePages(Flags:dict, PagesPaths:dict, LimitFiles, SiteTemplate, GlobalMacros, PoolSize:int): + MultiprocPages = [] + for Type in ('Page', 'Post'): + Files, PathPrefix = {"Page": [PagesPaths['Pages'], ''], "Post": [PagesPaths['Posts'], 'Posts/']}[Type] + for i, File in enumerate(Files): + TempPath = f"{PathPrefix}{File}" + LightRun = False if LimitFiles == False or TempPath in LimitFiles else True + MultiprocPages += [{'Flags': Flags, 'Page': [f"{Type}s/{File}", TempPath, Type, None], 'Template': SiteTemplate, 'GlobalMacros': GlobalMacros, 'LightRun': LightRun}] + return DoMultiProc(MultiprocPagePreprocessor, MultiprocPages, PoolSize, True) + +def WriteProcessedPages(Flags:dict, Pages:list, Categories, ConfMenu, Snippets, LimitFiles, PoolSize:int, Locale:dict): + MultiprocPages = [] + for i, Page in enumerate(Pages): + MultiprocPages += [{'Flags': Flags, 'Page': Page, 'Pages': Pages, 'Categories': Categories, 'LimitFiles': LimitFiles, 'Snippets': Snippets, 'ConfMenu': ConfMenu, 'Locale': Locale}] + return DoMultiProc(MultiprocHandlePage, MultiprocPages, PoolSize, True) + +def MakeSite(Flags:dict, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale:dict, Threads): + Pages, MadePages, Categories = [], [], {} PoolSize = cpu_count() if Threads <= 0 else Threads + + f = NameSpace(Flags) OutDir, MarkdownExts, Sorting = Flags['OutDir'], Flags['MarkdownExts'], Flags['Sorting'] SiteName, BlogName, SiteTagline = Flags['SiteName'], Flags['BlogName'], Flags['SiteTagline'] SiteTemplate, SiteLang = Flags['SiteTemplate'], Flags['SiteLang'] @@ -331,74 +389,29 @@ def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale, Thread ImgAltToTitle, ImgTitleToAlt = Flags['ImgAltToTitle'], Flags['ImgTitleToAlt'] DynamicParts, DynamicPartsText, StaticPartsText, TemplatesText = Flags['DynamicParts'], Snippets['DynamicParts'], Snippets['StaticParts'], Snippets['Templates'] - for Ext in FileExtensions['Pages']: - for File in Path('Pages').rglob(f"*.{Ext}"): - PagesPaths += [FileToStr(File, 'Pages/')] - for File in Path('Posts').rglob(f"*.{Ext}"): - PostsPaths += [FileToStr(File, 'Posts/')] - logging.info(f"Pages Found: {len(PagesPaths+PostsPaths)}") + logging.info("Finding Pages") + PagesPaths = FindPagesPaths() + logging.info(f"Pages Found: {len(PagesPaths['Pages']+PagesPaths['Posts'])}") - PagesPaths = FileNameDateSort(PagesPaths) - if Sorting['Pages'] == 'Inverse': - PagesPaths.reverse() - PostsPaths = FileNameDateSort(PostsPaths) - if Sorting['Posts'] == 'Inverse': - PostsPaths.reverse() + logging.info("Reordering Pages") + PagesPaths = ReorderPagesPaths(PagesPaths, f.Sorting) logging.info("Preprocessing Source Pages") - MultiprocPages = [] - for Type in ['Page', 'Post']: - if Type == 'Page': - Files = PagesPaths - PathPrefix = '' - elif Type == 'Post': - Files = PostsPaths - PathPrefix = 'Posts/' - for i,File in enumerate(Files): - TempPath = f"{PathPrefix}{File}" - LightRun = False if LimitFiles == False or TempPath in LimitFiles else True - MultiprocPages += [{'Process':{'Num':i, 'Count':len(Files)}, 'Path':f"{Type}s/{File}", 'TempPath':TempPath, 'Type':Type, 'Template':SiteTemplate, 'SiteRoot':SiteRoot, 'GlobalMacros':GlobalMacros, 'CategoryUncategorized':CategoryUncategorized, 'LightRun':LightRun}] - os.system('printf "["') - with Pool(PoolSize) as MultiprocPool: - Pages = MultiprocPool.map(MultiprocPagePreprocessor, MultiprocPages) - os.system('printf "]\n"') # Make newline after percentage dots + Pages = PreprocessSourcePages(Flags, PagesPaths, LimitFiles, SiteTemplate, GlobalMacros, PoolSize) for File, Content, Titles, Meta in Pages: for Cat in Meta['Categories']: Categories.update({Cat:''}) + PugCompileList(OutDir, Pages, LimitFiles) - if Categories: + if Categories or f.CategoriesAutomatic: logging.info("Generating Category Lists") - for Cat in Categories: - for Type in ('Page', 'Post'): - Categories[Cat] += GetHTMLPagesList( - Pages=Pages, - BlogName=BlogName, - SiteRoot=SiteRoot, - PathPrefix=GetPathLevels('Categories/'), - Type=Type, - Category=Cat, - For='Categories', - MarkdownExts=MarkdownExts, - MenuStyle='Flat') - - if AutoCategories: - Dir = f"{OutDir}/Categories" - for Cat in Categories: - Exists = False - for File in Path(Dir).rglob(str(Cat)+'.*'): - Exists = True - break - if not Exists: - File = f"Categories/{Cat}.md" - FilePath = f"{OutDir}/{File}" - WriteFile(FilePath, CategoryPageTemplate.format(Name=Cat)) - _, Content, Titles, Meta = PagePreprocessor(FilePath, FilePath, Type, SiteTemplate, SiteRoot, GlobalMacros, CategoryUncategorized, LightRun=LightRun) - Pages += [[File, Content, Titles, Meta]] + Categories = PopulateCategoryLists(Flags, Pages, Categories) + Pages += MakeAutoCategories(Flags, Categories) #logging.info("Building the HTML Search Page") - #Pages += [PagePreprocessor(Path='Search.html', TempPath='Search.html', Type='Page', SiteTemplate=SiteTemplate, SiteRoot=SiteRoot, GlobalMacros=GlobalMacros, CategoryUncategorized=CategoryUncategorized, LightRun=LightRun, Content=BuildPagesSearch(Flags, Pages))] + #Pages += [PagePreprocessor(Flags, Path='Search.html', TempPath='Search.html', Type='Page', SiteTemplate=SiteTemplate, GlobalMacros=GlobalMacros, LightRun=LightRun, Content=BuildPagesSearch(Flags, Pages))] for i,e in enumerate(ConfMenu): for File, Content, Titles, Meta in Pages: @@ -407,13 +420,7 @@ def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale, Thread ConfMenu[i] = None logging.info("Writing Pages") - MultiprocPages = [] - for i,Page in enumerate(Pages): - MultiprocPages += [{'Process':{'Num':i, 'Count':len(Pages)}, 'Flags':Flags, 'Page':Page, 'Pages':Pages, 'Categories':Categories, 'LimitFiles':LimitFiles, 'Snippets':Snippets, 'ConfMenu':ConfMenu, 'Locale':Locale}] - os.system('printf "["') - with Pool(PoolSize) as MultiprocPool: - MadePages = MultiprocPool.map(MultiprocHandlePage, MultiprocPages) - os.system('printf "]\n"') # Make newline after percentage dots + MadePages = WriteProcessedPages(Flags, Pages, Categories, ConfMenu, Snippets, LimitFiles, PoolSize, Locale) # Do page transclusions here (?) #while True: diff --git a/App/Source/Modules/Sitemap.py b/App/Source/Modules/Sitemap.py index da43152..8f3e7e8 100644 --- a/App/Source/Modules/Sitemap.py +++ b/App/Source/Modules/Sitemap.py @@ -11,7 +11,7 @@ from urllib.parse import quote as URLEncode from Modules.HTML import * from Modules.Utils import * -def MakeSitemap(Flags, Pages): +def MakeSitemap(Flags:dict, Pages:list): Map = '' Domain = Flags['SiteDomain'] + '/' if Flags['SiteDomain'] else '' for File, Content, Titles, Meta, ContentHtml, SlimHtml, Description, Image in Pages: diff --git a/App/Source/Modules/Social.py b/App/Source/Modules/Social.py new file mode 100644 index 0000000..aad717c --- /dev/null +++ b/App/Source/Modules/Social.py @@ -0,0 +1,48 @@ +""" ================================== | +| This file is part of | +| staticoso | +| Just a simple Static Site Generator | +| | +| Licensed under the AGPLv3 license | +| Copyright (C) 2022-2023, OctoSpacc | +| ================================== """ + +from Modules.Logging import * +from Modules.Utils import * + +try: + from Modules.ActivityPub import * + ActivityPub = True +except: + logging.warning("⚠ Can't load the ActivityPub module. Its use is disabled. Make sure the 'requests' library is installed.") + ActivityPub = False + +def ApplySocialIntegrations(Flags, Pages, LimitFiles, Locale): + f = NameSpace(Flags) + FinalPaths = [] + + if ActivityPub and f.MastodonURL and f.MastodonToken and f.SiteDomain: + logging.info("Mastodon Operations") + MastodonPosts = MastodonShare(Flags, Pages, Locale) + else: + MastodonPosts = [] + + for File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image in Pages: + if IsLightRun(File, LimitFiles): + continue + File = f'{f.OutDir}/{StripExt(File)}.html' + Content = ReadFile(File) + Post = '' + for p in MastodonPosts: + if p['Link'] == SiteDomain + '/' + File[len(f'{f.OutDir}/'):]: + Post = HTMLCommentsBlock.format( + StrComments=Locale['Comments'], + StrOpen=Locale['OpenInNewTab'], + URL=p['Post']) + break + #Content = ReplWithEsc(Content, '[staticoso:Comments]', Post) + Content = ReplWithEsc(Content, '', Post) + WriteFile(File, Content) + FinalPaths += [File] + + return FinalPaths diff --git a/App/Source/Modules/Utils.py b/App/Source/Modules/Utils.py index 4a13810..56f4d1f 100644 --- a/App/Source/Modules/Utils.py +++ b/App/Source/Modules/Utils.py @@ -10,7 +10,9 @@ import json import os from datetime import datetime +from multiprocessing import Pool, cpu_count from pathlib import Path +from types import SimpleNamespace from Modules.Globals import * def SureList(e): @@ -36,14 +38,14 @@ def WriteFile(p, c, m='w'): logging.error(f"[E] Error writing file {p}") return False -def FileToStr(File, Truncate=''): +def FileToStr(File:str, Truncate:str=''): return str(File)[len(Truncate):] # With shutil.copytree copy only folder struct, no files; https://stackoverflow.com/a/15664273 -def IgnoreFiles(Dir, Files): +def IgnoreFiles(Dir:str, Files): return [f for f in Files if os.path.isfile(os.path.join(Dir, f))] -def LoadFromDir(Dir, Matchs): +def LoadFromDir(Dir:str, Matchs): Contents = {} Matchs = SureList(Matchs) for Match in Matchs: @@ -52,10 +54,10 @@ def LoadFromDir(Dir, Matchs): Contents.update({File: ReadFile(f"{Dir}/{File}")}) return Contents -def mkdirps(Dir): +def mkdirps(Dir:str): return Path(Dir).mkdir(parents=True, exist_ok=True) -def StripExt(Path): +def StripExt(Path:str): return ".".join(Path.split('.')[:-1]) def UndupeStr(Str, Known, Split): @@ -174,3 +176,28 @@ def PrintProcPercentDots(Proc, DivMult=1): os.system('printf "="') # Using sys shell since for some reason print() without newline breaks here (doesn't print everytime) return True return False + +def NameSpace(From): + return SimpleNamespace(**From) + +def DoMultiProc(Funct, ArgsCollection:list, Threads:int=cpu_count(), Progress:bool=False): + # The function should simply be like this + # def Funct(Args:dict): + # # Print the percentage dots, if needed + # PrintProcPercentDots(Args['Process']) + # # Call the real function that handles a single process at a time + # return Work(Args['a'], ...) + #if not Threads: + # Threads = cpu_count() + FinalArgsCollection = [] + for Index, Args in enumerate(ArgsCollection): + FinalArgsCollection.append(Args) + FinalArgsCollection[Index].update({"Process": {"Num": Index, "Count": len(ArgsCollection)}}) + Results = [] + if Progress: + os.system('printf "["') # Using system print because (see PrintProcPercentDots()) + with Pool(Threads) as MultiprocPool: + Results = MultiprocPool.map(Funct, FinalArgsCollection) + if Progress: + os.system('printf "]\n"') # Newline after percentage dots + return Results