From a342f88e3c8f4bae45c8dc720e70620048e470dc Mon Sep 17 00:00:00 2001 From: octospacc Date: Fri, 24 Feb 2023 15:29:36 +0100 Subject: [PATCH] Generate HTML search with template; More Refactoring of the main workflows --- App/Source/Build.py | 136 ++++++++++++++-------------- App/Source/Modules/Assets.py | 2 +- App/Source/Modules/Feed.py | 1 + App/Source/Modules/Gemini.py | 2 +- App/Source/Modules/Globals.py | 26 +++++- App/Source/Modules/Meta.py | 30 ++----- App/Source/Modules/Site.py | 164 +++++++++++++++++----------------- App/Source/Modules/Sitemap.py | 23 +---- App/Source/Modules/Utils.py | 42 ++++----- 9 files changed, 206 insertions(+), 220 deletions(-) diff --git a/App/Source/Build.py b/App/Source/Build.py index 323d4d6..508c1e0 100644 --- a/App/Source/Build.py +++ b/App/Source/Build.py @@ -26,25 +26,23 @@ from Modules.Sitemap import * from Modules.Social import * from Modules.Utils import * -def ResetOutDir(OutDir): +def ResetOutDir(OutDir:str): for e in (OutDir, f'{OutDir}.Content', f'{OutDir}.gmi'): try: shutil.rmtree(e) except FileNotFoundError: pass -def DelTmp(OutDir): +def DelTmp(OutDir:str): for Ext in FileExtensions['Tmp']: - for File in Path(OutDir).rglob(f"*.{Ext}"): + for File in Path(OutDir).rglob(f'*.{Ext}'): os.remove(File) - for Dir in (OutDir, f"{OutDir}.gmi"): + for Dir in (OutDir, f'{OutDir}.gmi'): for File in Path(Dir).rglob('*.tmp'): os.remove(File) -def SetSorting(Sorting): - Default = { - 'Pages':'Standard', - 'Posts':'Inverse'} +def SetSorting(Sorting:dict): + Default = {"Pages": "Standard", "Posts": "Inverse"} for i in Default: if i not in Sorting: Sorting.update({i:Default[i]}) @@ -65,14 +63,14 @@ def GetConfMenu(Entries, MarkdownExts): Menu[int(i)] = e return Menu -def CheckSafeOutDir(OutDir): +def CheckSafeOutDir(OutDir:str): InDir = os.path.realpath(os.getcwd()) OutDir = os.path.realpath(OutDir) OutFolder = OutDir.split('/')[-1] - if InDir == OutDir: + if InDir.lower() == OutDir.lower(): logging.error(f"⛔ Output and Input directories ({OutDir}) can't be the same. Exiting.") exit(1) - elif OutFolder in ReservedPaths and f"{InDir}/{OutFolder}" == OutDir: + elif OutFolder.lower() in ReservedPaths and f"{InDir.lower()}/{OutFolder.lower()}" == OutDir.lower(): logging.error(f"⛔ Output directory {OutDir} can't be a reserved subdirectory of the Input. Exiting.") exit(1) @@ -98,21 +96,34 @@ def GetModifiedFiles(OutDir): Mod += [File['Tmp']] return Mod -def WriteRedirects(Flags, Pages, FinalPaths, Locale): - OutDir, SiteName, SiteDomain = Flags['OutDir'], Flags['SiteName'], Flags['SiteDomain'] - for File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image in Pages: - for URL in Meta['URLs']: - DestFile = f"{OutDir}/{URL}" +def WriteRedirects(Flags:dict, Pages:list, FinalPaths, Locale:dict): + SiteName = Flags['SiteName'] + for Page in Pages: + for URL in Page['Meta']['URLs']: + DestFile = f'{Flags["OutDir"]}/{URL}' if DestFile not in FinalPaths: - DestURL = f"{GetPathLevels(URL)}{StripExt(File)}.html" + DestURL = f'{GetPathLevels(URL)}{StripExt(Page["File"])}.html' mkdirps(os.path.dirname(DestFile)) WriteFile(DestFile, RedirectPageTemplate.format( - SiteDomain=SiteDomain, + SiteDomain=Flags['SiteDomain'], DestURL=DestURL, - TitlePrefix=f"{SiteName} - " if SiteName else '', + TitlePrefix=f'{SiteName} - ' if SiteName else '', StrClick=Locale['ClickHere'], StrRedirect=Locale['IfNotRedirected'])) +def CopyBaseFiles(Flags:dict): + f = NameSpace(Flags) + Have = {"Pages": False, "Posts": False} + Prefix = {"Pages": "", "Posts": "/Posts"} + for Type in ('Pages', 'Posts'): + if os.path.isdir(Type): + Have[Type] = True + shutil.copytree(Type, f'{f.OutDir}{Prefix[Type]}', dirs_exist_ok=True) + shutil.copytree(Type, f'{f.OutDir}.Content{Prefix[Type]}', dirs_exist_ok=True) + if f.GemtextOutput: + shutil.copytree('Posts', f'{f.OutDir}.gmi{Prefix[Type]}', ignore=IgnoreFiles, dirs_exist_ok=True) + return Have['Pages'] or Have['Posts'] + def BuildMain(Args, FeedEntries): Flags, Snippets = {}, {} HavePages, HavePosts = False, False @@ -136,48 +147,48 @@ def BuildMain(Args, FeedEntries): Threads = Args.Threads if Args.Threads else DefConf['Threads'] DiffBuild = Args.DiffBuild if Args.DiffBuild else DefConf['DiffBuild'] - BlogName = Flags['BlogName'] = OptChoose('', Args.BlogName, ReadConf(SiteConf, 'Site', 'BlogName')) - SiteTagline = Flags['SiteTagline'] = OptChoose('', Args.SiteTagline, ReadConf(SiteConf, 'Site', 'Tagline')) - SiteTemplate = Flags['SiteTemplate'] = DefConfOptChoose('SiteTemplate', Args.SiteTemplate, ReadConf(SiteConf, 'Site', 'Template')) + Flags['BlogName'] = OptChoose('', Args.BlogName, ReadConf(SiteConf, 'Site', 'BlogName')) + Flags['SiteTagline'] = OptChoose('', Args.SiteTagline, ReadConf(SiteConf, 'Site', 'Tagline')) + Flags['SiteTemplate'] = DefConfOptChoose('SiteTemplate', Args.SiteTemplate, ReadConf(SiteConf, 'Site', 'Template')) SiteDomain = Flags['SiteDomain'] = OptChoose('', Args.SiteDomain, ReadConf(SiteConf, 'Site', 'Domain')) - SiteRoot = Flags['SiteRoot'] = OptChoose('/', Args.SiteRoot, ReadConf(SiteConf, 'Site', 'Root')) + Flags['SiteRoot'] = OptChoose('/', Args.SiteRoot, ReadConf(SiteConf, 'Site', 'Root')) SiteLang = Flags['SiteLang'] = DefConfOptChoose('SiteLang', Args.SiteLanguage, ReadConf(SiteConf, 'Site', 'Language')) Sorting = Flags['Sorting'] = literal_eval(OptChoose('{}', Args.Sorting, ReadConf(SiteConf, 'Site', 'Sorting'))) Sorting = Flags['Sorting'] = SetSorting(Sorting) - NoScripts = Flags['NoScripts'] = StrBoolChoose(False, Args.NoScripts, ReadConf(SiteConf, 'Site', 'NoScripts')) - FolderRoots = Flags['FolderRoots'] = literal_eval(Args.FolderRoots) if Args.FolderRoots else {} + Flags['NoScripts'] = StrBoolChoose(False, Args.NoScripts, ReadConf(SiteConf, 'Site', 'NoScripts')) + Flags['FolderRoots'] = literal_eval(Args.FolderRoots) if Args.FolderRoots else {} - ActivityPubTypeFilter = Flags['ActivityPubTypeFilter'] = DefConfOptChoose('ActivityPubTypeFilter', Args.ActivityPubTypeFilter, ReadConf(SiteConf, 'ActivityPub', 'TypeFilter')) - ActivityPubHoursLimit = Flags['ActivityPubHoursLimit'] = DefConfOptChoose('ActivityPubHoursLimit', Args.ActivityPubHoursLimit, ReadConf(SiteConf, 'ActivityPub', 'HoursLimit')) + Flags['ActivityPubTypeFilter'] = DefConfOptChoose('ActivityPubTypeFilter', Args.ActivityPubTypeFilter, ReadConf(SiteConf, 'ActivityPub', 'TypeFilter')) + Flags['ActivityPubHoursLimit'] = DefConfOptChoose('ActivityPubHoursLimit', Args.ActivityPubHoursLimit, ReadConf(SiteConf, 'ActivityPub', 'HoursLimit')) - MastodonURL = Flags['MastodonURL'] = OptChoose('', Args.MastodonURL, ReadConf(SiteConf, 'Mastodon', 'URL')) - MastodonToken = Flags['MastodonToken'] = OptChoose('', Args.MastodonToken, ReadConf(SiteConf, 'Mastodon', 'Token')) + Flags['MastodonURL'] = OptChoose('', Args.MastodonURL, ReadConf(SiteConf, 'Mastodon', 'URL')) + Flags['MastodonToken'] = OptChoose('', Args.MastodonToken, ReadConf(SiteConf, 'Mastodon', 'Token')) MarkdownExts = Flags['MarkdownExts'] = literal_eval(OptionChoose(str(MarkdownExtsDefault), Args.MarkdownExts, ReadConf(SiteConf, 'Markdown', 'Exts'))) - SitemapOutput = Flags['SitemapOutput'] = StrBoolChoose(True, Args.SitemapOutput, ReadConf(SiteConf, 'Sitemap', 'Output')) + Flags['SitemapOutput'] = StrBoolChoose(True, Args.SitemapOutput, ReadConf(SiteConf, 'Sitemap', 'Output')) - MinifyOutput = Flags['MinifyOutput'] = StrBoolChoose(False, Args.MinifyOutput, ReadConf(SiteConf, 'Minify', 'Output')) - MinifyAssets = Flags['MinifyAssets'] = StrBoolChoose(False, Args.MinifyAssets, ReadConf(SiteConf, 'Minify', 'Assets')) - MinifyKeepComments = Flags['MinifyKeepComments'] = StrBoolChoose(False, Args.MinifyKeepComments, ReadConf(SiteConf, 'Minify', 'KeepComments')) + Flags['MinifyOutput'] = StrBoolChoose(False, Args.MinifyOutput, ReadConf(SiteConf, 'Minify', 'Output')) + Flags['MinifyAssets'] = StrBoolChoose(False, Args.MinifyAssets, ReadConf(SiteConf, 'Minify', 'Assets')) + Flags['MinifyKeepComments'] = StrBoolChoose(False, Args.MinifyKeepComments, ReadConf(SiteConf, 'Minify', 'KeepComments')) - ImgAltToTitle = Flags['ImgAltToTitle'] = StrBoolChoose(True, Args.ImgAltToTitle, ReadConf(SiteConf, 'Site', 'ImgAltToTitle')) - ImgTitleToAlt = Flags['ImgTitleToAlt'] = StrBoolChoose(False, Args.ImgTitleToAlt, ReadConf(SiteConf, 'Site', 'ImgTitleToAlt')) - HTMLFixPre = Flags['HTMLFixPre'] = StrBoolChoose(False, Args.HTMLFixPre, ReadConf(SiteConf, 'Site', 'HTMLFixPre')) + Flags['ImgAltToTitle'] = StrBoolChoose(True, Args.ImgAltToTitle, ReadConf(SiteConf, 'Site', 'ImgAltToTitle')) + Flags['ImgTitleToAlt'] = StrBoolChoose(False, Args.ImgTitleToAlt, ReadConf(SiteConf, 'Site', 'ImgTitleToAlt')) + Flags['HTMLFixPre'] = StrBoolChoose(False, Args.HTMLFixPre, ReadConf(SiteConf, 'Site', 'HTMLFixPre')) - CategoriesAutomatic = Flags['CategoriesAutomatic'] = StrBoolChoose(False, Args.CategoriesAutomatic, ReadConf(SiteConf, 'Categories', 'Automatic')) - CategoriesUncategorized = Flags['CategoriesUncategorized'] = DefConfOptChoose('CategoriesUncategorized', Args.CategoriesUncategorized, ReadConf(SiteConf, 'Categories', 'Uncategorized')) + Flags['CategoriesAutomatic'] = StrBoolChoose(False, Args.CategoriesAutomatic, ReadConf(SiteConf, 'Categories', 'Automatic')) + Flags['CategoriesUncategorized'] = DefConfOptChoose('CategoriesUncategorized', Args.CategoriesUncategorized, ReadConf(SiteConf, 'Categories', 'Uncategorized')) - GemtextOutput = Flags['GemtextOutput'] = StrBoolChoose(False, Args.GemtextOutput, ReadConf(SiteConf, 'Gemtext', 'Output')) - GemtextHeader = Flags['GemtextHeader'] = Args.GemtextHeader if Args.GemtextHeader else ReadConf(SiteConf, 'Gemtext', 'Header') if ReadConf(SiteConf, 'Gemtext', 'Header') else f"# {SiteName}\n\n" if SiteName else '' + Flags['GemtextOutput'] = StrBoolChoose(False, Args.GemtextOutput, ReadConf(SiteConf, 'Gemtext', 'Output')) + Flags['GemtextHeader'] = Args.GemtextHeader if Args.GemtextHeader else ReadConf(SiteConf, 'Gemtext', 'Header') if ReadConf(SiteConf, 'Gemtext', 'Header') else f"# {SiteName}\n\n" if SiteName else '' - FeedCategoryFilter = Flags['FeedCategoryFilter'] = DefConfOptChoose('FeedCategoryFilter', Args.FeedCategoryFilter, ReadConf(SiteConf, 'Feed', 'CategoryFilter')) + Flags['FeedCategoryFilter'] = DefConfOptChoose('FeedCategoryFilter', Args.FeedCategoryFilter, ReadConf(SiteConf, 'Feed', 'CategoryFilter')) FeedEntries = Flags['FeedEntries'] = int(FeedEntries) if (FeedEntries or FeedEntries == 0) and FeedEntries != 'Default' else int(ReadConf(SiteConf, 'Feed', 'Entries')) if ReadConf(SiteConf, 'Feed', 'Entries') else DefConf['FeedEntries'] - JournalRedirect = Flags["JournalRedirect"] = StrBoolChoose(DefConf["JournalRedirect"], Args.JournalRedirect, ReadConf(SiteConf, 'Journal', 'Redirect')) + Flags["JournalRedirect"] = StrBoolChoose(DefConf["JournalRedirect"], Args.JournalRedirect, ReadConf(SiteConf, 'Journal', 'Redirect')) - DynamicParts = Flags['DynamicParts'] = literal_eval(OptionChoose('{}', Args.DynamicParts, ReadConf(SiteConf, 'Site', 'DynamicParts'))) + Flags['DynamicParts'] = literal_eval(OptionChoose('{}', Args.DynamicParts, ReadConf(SiteConf, 'Site', 'DynamicParts'))) DynamicPartsText = Snippets['DynamicParts'] = LoadFromDir('DynamicParts', ['*.htm', '*.html']) StaticPartsText = Snippets['StaticParts'] = LoadFromDir('StaticParts', ['*.htm', '*.html']) TemplatesText = Snippets['Templates'] = LoadFromDir('Templates', ['*.htm', '*.html']) @@ -199,25 +210,13 @@ def BuildMain(Args, FeedEntries): ResetOutDir(OutDir) LimitFiles = False - if os.path.isdir('Pages'): - HavePages = True - shutil.copytree('Pages', OutDir, dirs_exist_ok=True) - shutil.copytree('Pages', f'{OutDir}.Content', dirs_exist_ok=True) - if Flags['GemtextOutput']: - shutil.copytree('Pages', f'{OutDir}.gmi', ignore=IgnoreFiles, dirs_exist_ok=True) - if os.path.isdir('Posts'): - HavePosts = True - shutil.copytree('Posts', f'{OutDir}/Posts', dirs_exist_ok=True) - shutil.copytree('Posts', f'{OutDir}.Content/Posts', dirs_exist_ok=True) - if Flags['GemtextOutput']: - shutil.copytree('Posts', f'{OutDir}.gmi/Posts', ignore=IgnoreFiles, dirs_exist_ok=True) - - if not (HavePages or HavePosts): + logging.info("Reading Base Files") + if not (CopyBaseFiles(Flags)): logging.error("⛔ No Pages or posts found. Nothing to do, exiting!") exit(1) logging.info("Generating HTML") - Pages = MakeSite( + DictPages = MakeSite( Flags=Flags, LimitFiles=LimitFiles, Snippets=Snippets, @@ -226,34 +225,35 @@ def BuildMain(Args, FeedEntries): Locale=Locale, Threads=Threads) - # REFACTOR: The functions below are still not changed to accept a Page as Dict - for i, e in enumerate(Pages): - Pages[i] = list(e.values()) + # REFACTOR: Some functions below are still not changed to accept a Page as Dict, so let's convert to Lists + ListPages = DictPages.copy() + for i, e in enumerate(ListPages): + ListPages[i] = list(e.values()) if FeedEntries != 0: logging.info("Generating Feeds") for FeedType in (True, False): - MakeFeed(Flags, Pages, FeedType) + MakeFeed(Flags, ListPages, FeedType) logging.info("Applying Social Integrations") - FinalPaths = ApplySocialIntegrations(Flags, Pages, LimitFiles, Locale) + FinalPaths = ApplySocialIntegrations(Flags, ListPages, LimitFiles, Locale) logging.info("Creating Redirects") - WriteRedirects(Flags, Pages, FinalPaths, Locale) + WriteRedirects(Flags, DictPages, FinalPaths, Locale) logging.info("Building HTML Search Page") - WriteFile(f'{OutDir}/Search.html', BuildPagesSearch(Flags, Pages)) + WriteFile(f'{OutDir}/Search.html', BuildPagesSearch(Flags, DictPages, TemplatesText[Flags['SiteTemplate']], Snippets, Locale)) if Flags['GemtextOutput']: logging.info("Generating Gemtext") - GemtextCompileList(Flags, Pages, LimitFiles) + GemtextCompileList(Flags, ListPages, LimitFiles) logging.info("Cleaning Temporary Files") DelTmp(OutDir) if Flags['SitemapOutput']: logging.info("Generating Sitemap") - MakeSitemap(Flags, Pages) + MakeSitemap(Flags, DictPages) logging.info("Preparing Assets") PrepareAssets(Flags) @@ -268,7 +268,7 @@ if __name__ == '__main__': Parser.add_argument('--OutputDir', type=str) #Parser.add_argument('--InputDir', type=str) Parser.add_argument('--Sorting', type=str) - Parser.add_argument('--SiteLang', type=str) # DEPRECATED + #Parser.add_argument('--SiteLang', type=str) # DEPRECATED Parser.add_argument('--SiteLanguage', type=str) Parser.add_argument('--SiteRoot', type=str) Parser.add_argument('--SiteName', type=str) diff --git a/App/Source/Modules/Assets.py b/App/Source/Modules/Assets.py index be8a2ae..640a9e6 100644 --- a/App/Source/Modules/Assets.py +++ b/App/Source/Modules/Assets.py @@ -14,7 +14,7 @@ from Modules.Utils import * from Libs import rcssmin cssmin = rcssmin._make_cssmin(python_only=True) -def PrepareAssets(Flags): +def PrepareAssets(Flags:dict): f = NameSpace(Flags) if f.MinifyAssets: shutil.copytree('Assets', f.OutDir, ignore=IgnoreFiles, dirs_exist_ok=True) diff --git a/App/Source/Modules/Feed.py b/App/Source/Modules/Feed.py index 1bfd133..4baff75 100644 --- a/App/Source/Modules/Feed.py +++ b/App/Source/Modules/Feed.py @@ -13,6 +13,7 @@ from Libs.feedgen.feed import FeedGenerator from Modules.Utils import * def MakeFeed(Flags:dict, Pages:list, FullSite=False): + f = NameSpace(Flags) CategoryFilter = Flags['FeedCategoryFilter'] MaxEntries = Flags['FeedEntries'] diff --git a/App/Source/Modules/Gemini.py b/App/Source/Modules/Gemini.py index be691f3..1bcd37a 100644 --- a/App/Source/Modules/Gemini.py +++ b/App/Source/Modules/Gemini.py @@ -47,7 +47,7 @@ def GemtextCompileList(Flags:dict, Pages:list, LimitFiles): Gemtext += Line + '\n' WriteFile(Dst, Flags['GemtextHeader'] + Gemtext) -def FindEarliest(Str, Items): +def FindEarliest(Str:str, Items:list): Pos, Item = 0, '' for Item in Items: Str.find(Item) diff --git a/App/Source/Modules/Globals.py b/App/Source/Modules/Globals.py index 3b5dc1a..eead700 100644 --- a/App/Source/Modules/Globals.py +++ b/App/Source/Modules/Globals.py @@ -7,7 +7,7 @@ | Copyright (C) 2022-2023, OctoSpacc | | ================================== """ -ReservedPaths = ('Site.ini', 'Assets', 'Pages', 'Posts', 'Templates', 'StaticParts', 'DynamicParts') +ReservedPaths = ('site.ini', 'assets', 'pages', 'posts', 'templates', 'staticparts', 'dynamicparts') FileExtensions = { 'Pages': ('htm', 'html', 'markdown', 'md', 'pug', 'txt'), 'HTML': ('.htm', '.html'), @@ -20,4 +20,26 @@ NegStrBools = ('false', 'no', 'off', '0', 'disabled') PageIndexStrPos = tuple(list(PosStrBools) + ['all', 'listed', 'indexed', 'unlinked']) PageIndexStrNeg = tuple(list(NegStrBools) + ['none', 'unlisted', 'unindexed', 'hidden']) -InternalMacrosWraps = [['[', ']'], ['<', '>']] +InternalMacrosWraps = (('[', ']'), ('<', '>')) + +PageMetaDefault = { + 'Template': None, # Replace with var + 'Head': '', + 'Style': '', + 'Type': None, # Replace with var + 'Index': 'Unspecified', + 'Feed': 'True', + 'Title': '', + 'HTMLTitle': '', + 'Description': '', + 'Image': '', + 'Macros': {}, + 'Categories': [], + 'URLs': [], + 'CreatedOn': '', + 'UpdatedOn': '', + 'EditedOn': '', + 'Order': None, + 'Language': None, + 'Downsync': None +} diff --git a/App/Source/Modules/Meta.py b/App/Source/Modules/Meta.py index f951b49..db26e47 100644 --- a/App/Source/Modules/Meta.py +++ b/App/Source/Modules/Meta.py @@ -8,6 +8,7 @@ | ================================== """ from Modules.Config import * +from Modules.Globals import * from Modules.Elements import * from Modules.HTML import * from Modules.Markdown import * @@ -124,32 +125,15 @@ def FindPreprocLine(Line:str, Meta, Macros): # IgnoreBlocksStart += [l] return (Meta, Macros, Changed) -def PagePreprocessor(Flags:dict, Page:list, SiteTemplate, GlobalMacros, LightRun:bool=False): - CategoryUncategorized = Flags['CategoriesUncategorized'] +def PagePreprocessor(Flags:dict, Page:list, GlobalMacros:dict, LightRun:bool=False): + f = NameSpace(Flags) Path, TempPath, Type, Content = Page File = ReadFile(Path) if not Content else Content Path = Path.lower() - Content, Titles, DashyTitles, HTMLTitlesFound, Macros, Meta, MetaDefault = '', [], [], False, '', '', { - 'Template': SiteTemplate, - 'Head': '', - 'Style': '', - 'Type': Type, - 'Index': 'Unspecified', - 'Feed': 'True', - 'Title': '', - 'HTMLTitle': '', - 'Description': '', - 'Image': '', - 'Macros': {}, - 'Categories': [], - 'URLs': [], - 'CreatedOn': '', - 'UpdatedOn': '', - 'EditedOn': '', - 'Order': None, - 'Language': None, - 'Downsync': None} + Content, Titles, DashyTitles, HTMLTitlesFound, Macros, Meta = '', [], [], False, '', '' + MetaDefault = PageMetaDefault.copy() + MetaDefault.update({"Template": f.SiteTemplate, "Type": Type}) # Find all positions of '', add them in a list=[[pos0,pos1,line0,line1],...] for l in File.splitlines(): ll = l.lstrip().rstrip() @@ -252,7 +236,7 @@ def PagePreprocessor(Flags:dict, Page:list, SiteTemplate, GlobalMacros, LightRun Meta['EditedOn'] = Meta['UpdatedOn'] if Meta['Index'].lower() in ('default', 'unspecified', 'categories'): if not Meta['Categories']: - Meta['Categories'] = [CategoryUncategorized] + Meta['Categories'] = [f.CategoriesUncategorized] if Meta['Type'].lower() == 'page': Meta['Index'] = 'Categories' elif Meta['Type'].lower() == 'post': diff --git a/App/Source/Modules/Site.py b/App/Source/Modules/Site.py index 3451a64..2603bc9 100644 --- a/App/Source/Modules/Site.py +++ b/App/Source/Modules/Site.py @@ -7,9 +7,8 @@ | Copyright (C) 2022-2023, OctoSpacc | | ================================== """ -import shutil from datetime import datetime -from multiprocessing import Pool, cpu_count +from multiprocessing import cpu_count from Modules.Config import * from Modules.Elements import * from Modules.Globals import * @@ -20,12 +19,8 @@ from Modules.Meta import * from Modules.Pug import * from Modules.Utils import * -def PatchHTML(Flags, File, HTML, StaticPartsText, DynamicParts, DynamicPartsText, HTMLPagesList, PagePath, Content, Titles, Meta, FolderRoots, Categories, Locale, LightRun): - SiteDomain = Flags['SiteDomain'] - SiteRoot = Flags['SiteRoot'] - SiteLang = Flags['SiteLang'] - SiteName = Flags['SiteName'] - BlogName = Flags['BlogName'] +def PatchHTML(Flags:dict, File, HTML:str, Snippets:dict, HTMLPagesList:str, PagePath:str, Content:str, Titles:list, Meta:dict, Categories, Locale:dict, LightRun): + f = NameSpace(Flags) HTMLTitles = FormatTitles(Titles) BodyDescription, BodyImage = '', '' @@ -57,7 +52,7 @@ def PatchHTML(Flags, File, HTML, StaticPartsText, DynamicParts, DynamicPartsText '': '', '': ''}) - Title = GetTitle(File.split('/')[-1], Meta, Titles, 'MetaTitle', BlogName) + Title = GetTitle(File.split('/')[-1], Meta, Titles, 'MetaTitle', f.BlogName) Description = GetDescription(Meta, BodyDescription, 'MetaDescription') Image = GetImage(Meta, BodyImage, 'MetaImage') ContentHeader = MakeContentHeader(Meta, Locale, MakeCategoryLine(File, Meta)) @@ -70,23 +65,23 @@ def PatchHTML(Flags, File, HTML, StaticPartsText, DynamicParts, DynamicPartsText if (Line.startswith('[staticoso:DynamicPart:') and Line.endswith(']')) or (Line.startswith('')): Path = Line[len('", Text) for i in range(2): - for e in StaticPartsText: - HTML = ReplWithEsc(HTML, f"[staticoso:StaticPart:{e}]", StaticPartsText[e]) - HTML = ReplWithEsc(HTML, f"", StaticPartsText[e]) + for e in Snippets['StaticParts']: + HTML = ReplWithEsc(HTML, f"[staticoso:StaticPart:{e}]", Snippets['StaticParts'][e]) + HTML = ReplWithEsc(HTML, f"", Snippets['StaticParts'][e]) if LightRun: HTML = None @@ -96,7 +91,7 @@ def PatchHTML(Flags, File, HTML, StaticPartsText, DynamicParts, DynamicPartsText #'': Meta['Head'], # #DEPRECATION # 'staticoso:Site:Menu': HTMLPagesList, - 'staticoso:Page:Lang': Meta['Language'] if Meta['Language'] else SiteLang, + 'staticoso:Page:Lang': Meta['Language'] if Meta['Language'] else f.SiteLang, 'staticoso:Page:Chapters': HTMLTitles, 'staticoso:Page:Title': Title, 'staticoso:Page:Description': Description, @@ -105,8 +100,8 @@ def PatchHTML(Flags, File, HTML, StaticPartsText, DynamicParts, DynamicPartsText 'staticoso:Page:Style': Meta['Style'], ################ 'staticoso:SiteMenu': HTMLPagesList, - 'staticoso:PageLang': Meta['Language'] if Meta['Language'] else SiteLang, - 'staticoso:PageLanguage': Meta['Language'] if Meta['Language'] else SiteLang, + 'staticoso:PageLang': Meta['Language'] if Meta['Language'] else f.SiteLang, + 'staticoso:PageLanguage': Meta['Language'] if Meta['Language'] else f.SiteLang, 'staticoso:PageSections': HTMLTitles, 'staticoso:PageTitle': Title, 'staticoso:PageDescription': Description, @@ -118,24 +113,25 @@ def PatchHTML(Flags, File, HTML, StaticPartsText, DynamicParts, DynamicPartsText # #DEPRECATION # 'staticoso:Page:Content': Content, 'staticoso:Page:ContentInfo': ContentHeader, - 'staticoso:Site:Name': SiteName, - 'staticoso:Site:AbsoluteRoot': SiteRoot, + 'staticoso:Site:Name': f.SiteName, + 'staticoso:Site:AbsoluteRoot': f.SiteRoot, 'staticoso:Site:RelativeRoot': RelativeRoot, ################ 'staticoso:PageContent': Content, 'staticoso:PageContentInfo': ContentHeader, 'staticoso:BuildTime': TimeNow, - 'staticoso:SiteDomain': SiteDomain, - 'staticoso:SiteName': SiteName, - 'staticoso:SiteAbsoluteRoot': SiteRoot, + 'staticoso:SiteDomain': f.SiteDomain, + 'staticoso:SiteName': f.SiteName, + 'staticoso:BlogName': f.BlogName, + 'staticoso:SiteAbsoluteRoot': f.SiteRoot, 'staticoso:SiteRelativeRoot': RelativeRoot, }, InternalMacrosWraps) for e in Meta['Macros']: HTML = ReplWithEsc(HTML, f"[:{e}:]", Meta['Macros'][e]) - for e in FolderRoots: + for e in f.FolderRoots: HTML = WrapDictReplWithEsc(HTML, { - f'staticoso:CustomPath:{e}': FolderRoots[e], - f'staticoso:Folder:{e}:AbsoluteRoot': FolderRoots[e], #DEPRECATED + f'staticoso:CustomPath:{e}': f.FolderRoots[e], + f'staticoso:Folder:{e}:AbsoluteRoot': f.FolderRoots[e], #DEPRECATED }, InternalMacrosWraps) for e in Categories: HTML = WrapDictReplWithEsc(HTML, { @@ -150,23 +146,23 @@ def PatchHTML(Flags, File, HTML, StaticPartsText, DynamicParts, DynamicPartsText # #DEPRECATION # '[staticoso:Page:Title]': Title, '[staticoso:Page:Description]': Description, - '[staticoso:Site:Name]': SiteName, - '[staticoso:Site:AbsoluteRoot]': SiteRoot, + '[staticoso:Site:Name]': f.SiteName, + '[staticoso:Site:AbsoluteRoot]': f.SiteRoot, '[staticoso:Site:RelativeRoot]': RelativeRoot, ################ '': Title, '': Description, - '': SiteDomain, - '': SiteName, - '': SiteRoot, + '': f.SiteDomain, + '': f.SiteName, + '': f.SiteRoot, '': RelativeRoot, }, InternalMacrosWraps) for e in Meta['Macros']: ContentHTML = ReplWithEsc(ContentHTML, f"[:{e}:]", Meta['Macros'][e]) - for e in FolderRoots: + for e in f.FolderRoots: ContentHTML = WrapDictReplWithEsc(ContentHTML, { - f'staticoso:CustomPath:{e}': FolderRoots[e], - f'staticoso:Folder:{e}:AbsoluteRoot': FolderRoots[e], #DEPRECATED + f'staticoso:CustomPath:{e}': f.FolderRoots[e], + f'staticoso:Folder:{e}:AbsoluteRoot': f.FolderRoots[e], #DEPRECATED }, InternalMacrosWraps) for e in Categories: ContentHTML = WrapDictReplWithEsc(ContentHTML, { @@ -177,24 +173,46 @@ def PatchHTML(Flags, File, HTML, StaticPartsText, DynamicParts, DynamicPartsText return HTML, ContentHTML, Description, Image -def HandlePage(Flags:dict, Page:list, Pages, Categories, LimitFiles, Snippets, ConfMenu, Locale:dict): +def BuildPagesSearch(Flags:dict, Pages:list, Template:str, Snippets:dict, Locale:dict): + SearchContent = '' + with open(f'{staticosoBaseDir()}Assets/PagesSearch.html', 'r') as File: + Base = File.read().split('{{PagesInject}}') + for Page in Pages: + SearchContent += f''' +
+ {Page["ContentHtml"]} +
+ ''' + return PatchHTML( + Flags=Flags, + File='Search.html', + HTML=Template, + Snippets=Snippets, + HTMLPagesList='', + PagePath='Search.html', + Content=Base[0] + SearchContent + Base[1], + Titles=[], + Meta=PageMetaDefault, + Categories=[], + Locale=Locale, + LightRun=False)[0] + +def HandlePage(Flags:dict, Page:list, Pages, Categories, LimitFiles, Snippets:dict, ConfMenu, Locale:dict): File, Content, Titles, Meta = Page - - OutDir, MarkdownExts, Sorting, MinifyKeepComments = Flags['OutDir'], Flags['MarkdownExts'], Flags['Sorting'], Flags['MinifyKeepComments'] - SiteName, BlogName, SiteTagline = Flags['SiteName'], Flags['BlogName'], Flags['SiteTagline'] - SiteTemplate, SiteLang = Flags['SiteTemplate'], Flags['SiteLang'] - SiteDomain, SiteRoot, FolderRoots = Flags['SiteDomain'], Flags['SiteRoot'], Flags['FolderRoots'] - AutoCategories, CategoryUncategorized = Flags['CategoriesAutomatic'], Flags['CategoriesUncategorized'] - ImgAltToTitle, ImgTitleToAlt = Flags['ImgAltToTitle'], Flags['ImgTitleToAlt'] - DynamicParts, DynamicPartsText, StaticPartsText, TemplatesText = Flags['DynamicParts'], Snippets['DynamicParts'], Snippets['StaticParts'], Snippets['Templates'] + f = NameSpace(Flags) + TemplatesText = Snippets['Templates'] FileLower = File.lower() - PagePath = f'{OutDir}/{StripExt(File)}.html' - ContentPagePath = f'{OutDir}.Content/{StripExt(File)}.html' + PagePath = f'{f.OutDir}/{StripExt(File)}.html' + ContentPagePath = f'{f.OutDir}.Content/{StripExt(File)}.html' LightRun = False if LimitFiles == False or File in LimitFiles else True if FileLower.endswith(FileExtensions['Markdown']): - Content = markdown(PagePostprocessor('md', Content, Meta), extensions=MarkdownExts) + Content = markdown(PagePostprocessor('md', Content, Meta), extensions=f.MarkdownExts) elif FileLower.endswith(('.pug')): Content = PagePostprocessor('pug', ReadFile(PagePath), Meta) elif FileLower.endswith(('.txt')): @@ -219,15 +237,12 @@ def HandlePage(Flags:dict, Page:list, Pages, Categories, LimitFiles, Snippets, C Flags, File=File, HTML=TemplatesText[Meta['Template']], - StaticPartsText=StaticPartsText, - DynamicParts=DynamicParts, - DynamicPartsText=DynamicPartsText, + Snippets=Snippets, HTMLPagesList=HTMLPagesList, - PagePath=PagePath[len(f"{OutDir}/"):], + PagePath=PagePath[len(f"{f.OutDir}/"):], Content=Content, Titles=Titles, Meta=Meta, - FolderRoots=FolderRoots, Categories=Categories, Locale=Locale, LightRun=LightRun) @@ -259,16 +274,16 @@ def HandlePage(Flags:dict, Page:list, Pages, Categories, LimitFiles, Snippets, C if Flags['MinifyOutput']: if not LightRun: - HTML = DoMinifyHTML(HTML, MinifyKeepComments) - ContentHTML = DoMinifyHTML(ContentHTML, MinifyKeepComments) + HTML = DoMinifyHTML(HTML, f.MinifyKeepComments) + ContentHTML = DoMinifyHTML(ContentHTML, f.MinifyKeepComments) if Flags['NoScripts'] and ('', f"""""") WriteFile(StripExt(PagePath)+'.Journal.html', HTML) - #return [File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image] - return {"File":File, "Content":Content, "Titles":Titles, "Meta":Meta, "ContentHtml":ContentHTML, "SlimHtml":SlimHTML, "Description":Description, "Image":Image} + return {"File": File, "Content": Content, "Titles": Titles, "Meta": Meta, "ContentHtml": ContentHTML, "SlimHtml": SlimHTML, "Description": Description, "Image": Image} def MultiprocPagePreprocessor(d:dict): - PrintProcPercentDots(d['Process'])#, 2) - return PagePreprocessor(d['Flags'], d['Page'], d['Template'], d['GlobalMacros'], d['LightRun']) + return PagePreprocessor(d['Flags'], d['Page'], d['GlobalMacros'], d['LightRun']) def MultiprocHandlePage(d:dict): - PrintProcPercentDots(d['Process']) return HandlePage(d['Flags'], d['Page'], d['Pages'], d['Categories'], d['LimitFiles'], d['Snippets'], d['ConfMenu'], d['Locale']) def FindPagesPaths(): @@ -356,18 +365,18 @@ def MakeAutoCategories(Flags:dict, Categories): File = f'Categories/{Cat}.md' FilePath = f'{OutDir}/{File}' WriteFile(FilePath, CategoryPageTemplate.format(Name=Cat)) - _, Content, Titles, Meta = PagePreprocessor(Flags, [FilePath, FilePath, Type, None], SiteTemplate, GlobalMacros, LightRun=LightRun) + _, Content, Titles, Meta = PagePreprocessor(Flags, [FilePath, FilePath, Type, None], GlobalMacros, LightRun=LightRun) Pages += [File, Content, Titles, Meta] return Pages -def PreprocessSourcePages(Flags:dict, PagesPaths:dict, LimitFiles, SiteTemplate, GlobalMacros, PoolSize:int): +def PreprocessSourcePages(Flags:dict, PagesPaths:dict, LimitFiles, GlobalMacros:dict, PoolSize:int): MultiprocPages = [] for Type in ('Page', 'Post'): Files, PathPrefix = {"Page": [PagesPaths['Pages'], ''], "Post": [PagesPaths['Posts'], 'Posts/']}[Type] for i, File in enumerate(Files): TempPath = f"{PathPrefix}{File}" LightRun = False if LimitFiles == False or TempPath in LimitFiles else True - MultiprocPages += [{'Flags': Flags, 'Page': [f"{Type}s/{File}", TempPath, Type, None], 'Template': SiteTemplate, 'GlobalMacros': GlobalMacros, 'LightRun': LightRun}] + MultiprocPages += [{'Flags': Flags, 'Page': [f"{Type}s/{File}", TempPath, Type, None], 'GlobalMacros': GlobalMacros, 'LightRun': LightRun}] return DoMultiProc(MultiprocPagePreprocessor, MultiprocPages, PoolSize, True) def WriteProcessedPages(Flags:dict, Pages:list, Categories, ConfMenu, Snippets, LimitFiles, PoolSize:int, Locale:dict): @@ -376,18 +385,10 @@ def WriteProcessedPages(Flags:dict, Pages:list, Categories, ConfMenu, Snippets, MultiprocPages += [{'Flags': Flags, 'Page': Page, 'Pages': Pages, 'Categories': Categories, 'LimitFiles': LimitFiles, 'Snippets': Snippets, 'ConfMenu': ConfMenu, 'Locale': Locale}] return DoMultiProc(MultiprocHandlePage, MultiprocPages, PoolSize, True) -def MakeSite(Flags:dict, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale:dict, Threads): +def MakeSite(Flags:dict, LimitFiles, Snippets, ConfMenu, GlobalMacros:dict, Locale:dict, Threads:int): Pages, MadePages, Categories = [], [], {} PoolSize = cpu_count() if Threads <= 0 else Threads - f = NameSpace(Flags) - OutDir, MarkdownExts, Sorting = Flags['OutDir'], Flags['MarkdownExts'], Flags['Sorting'] - SiteName, BlogName, SiteTagline = Flags['SiteName'], Flags['BlogName'], Flags['SiteTagline'] - SiteTemplate, SiteLang = Flags['SiteTemplate'], Flags['SiteLang'] - SiteDomain, SiteRoot, FolderRoots = Flags['SiteDomain'], Flags['SiteRoot'], Flags['FolderRoots'] - AutoCategories, CategoryUncategorized = Flags['CategoriesAutomatic'], Flags['CategoriesUncategorized'] - ImgAltToTitle, ImgTitleToAlt = Flags['ImgAltToTitle'], Flags['ImgTitleToAlt'] - DynamicParts, DynamicPartsText, StaticPartsText, TemplatesText = Flags['DynamicParts'], Snippets['DynamicParts'], Snippets['StaticParts'], Snippets['Templates'] logging.info("Finding Pages") PagesPaths = FindPagesPaths() @@ -397,14 +398,15 @@ def MakeSite(Flags:dict, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale:di PagesPaths = ReorderPagesPaths(PagesPaths, f.Sorting) logging.info("Preprocessing Source Pages") - Pages = PreprocessSourcePages(Flags, PagesPaths, LimitFiles, SiteTemplate, GlobalMacros, PoolSize) + Pages = PreprocessSourcePages(Flags, PagesPaths, LimitFiles, GlobalMacros, PoolSize) + PugCompileList(f.OutDir, Pages, LimitFiles) + + logging.info("Parsing Categories") for File, Content, Titles, Meta in Pages: for Cat in Meta['Categories']: Categories.update({Cat:''}) - PugCompileList(OutDir, Pages, LimitFiles) - if Categories or f.CategoriesAutomatic: logging.info("Generating Category Lists") Categories = PopulateCategoryLists(Flags, Pages, Categories) diff --git a/App/Source/Modules/Sitemap.py b/App/Source/Modules/Sitemap.py index 8f3e7e8..73db0e2 100644 --- a/App/Source/Modules/Sitemap.py +++ b/App/Source/Modules/Sitemap.py @@ -8,30 +8,11 @@ | ================================== """ from urllib.parse import quote as URLEncode -from Modules.HTML import * from Modules.Utils import * def MakeSitemap(Flags:dict, Pages:list): Map = '' Domain = Flags['SiteDomain'] + '/' if Flags['SiteDomain'] else '' - for File, Content, Titles, Meta, ContentHtml, SlimHtml, Description, Image in Pages: - File = f"{StripExt(File)}.html" - Map += Domain + URLEncode(File) + '\n' + for Page in Pages: + Map += Domain + URLEncode(f'{StripExt(Page["File"])}.html') + '\n' WriteFile(f"{Flags['OutDir']}/sitemap.txt", Map) - -def BuildPagesSearch(Flags:dict, Pages:list): - SearchContent = '' - with open(f'{staticosoBaseDir()}Assets/PagesSearch.html', 'r') as File: - Base = File.read().split('{{PagesInject}}') - for File, Content, Titles, Meta, ContentHtml, SlimHtml, Description, Image in Pages: - #for File, Content, Titles, Meta in Pages: - SearchContent += f''' -
- {ContentHtml} -
- ''' - return Base[0] + SearchContent + Base[1] diff --git a/App/Source/Modules/Utils.py b/App/Source/Modules/Utils.py index 56f4d1f..c5c7835 100644 --- a/App/Source/Modules/Utils.py +++ b/App/Source/Modules/Utils.py @@ -22,7 +22,7 @@ def SureList(e): def staticosoBaseDir(): return f"{os.path.dirname(os.path.abspath(__file__))}/../../" -def ReadFile(p, m='r'): +def ReadFile(p:str, m:str='r'): try: with open(p, m) as f: return f.read() @@ -42,10 +42,10 @@ def FileToStr(File:str, Truncate:str=''): return str(File)[len(Truncate):] # With shutil.copytree copy only folder struct, no files; https://stackoverflow.com/a/15664273 -def IgnoreFiles(Dir:str, Files): +def IgnoreFiles(Dir:str, Files:list): return [f for f in Files if os.path.isfile(os.path.join(Dir, f))] -def LoadFromDir(Dir:str, Matchs): +def LoadFromDir(Dir:str, Matchs:list): Contents = {} Matchs = SureList(Matchs) for Match in Matchs: @@ -70,26 +70,26 @@ def UndupeStr(Str, Known, Split): Str = Split.join(Sections) return Str -def DashifyStr(s, Limit=32): +def DashifyStr(s:str, Limit:int=32): Str = '' for c in s[:Limit].replace('\n','-').replace('\t','-').replace(' ','-'): if c.lower() in '0123456789qwfpbjluyarstgmneiozxcdvkh-': Str += c return '-' + Str -def GetPathLevels(Path, AsNum=False, Add=0, Sub=0): +def GetPathLevels(Path:str, AsNum:bool=False, Add:int=0, Sub:int=0): n = Path.count('/') + Add - Sub return n if AsNum else '../' * n # https://stackoverflow.com/a/34445090 -def FindAllIndex(Str, Sub): +def FindAllIndex(Str:str, Sub:str): i = Str.find(Sub) while i != -1: yield i i = Str.find(Sub, i+1) # Replace substrings in a string, except when an escape char is prepended -def ReplWithEsc(Str, Find, Repl, Esc='\\'): +def ReplWithEsc(Str:str, Find:str, Repl:str, Esc:str='\\'): New = '' Sects = Str.split(Find) for i,e in enumerate(Sects): @@ -118,7 +118,7 @@ def WrapDictReplWithEsc(Str:str, Dict:dict, Wraps:list=[], Esc:str='\\'): NewDict.update({f'{Wrap[0]}{Item}{Wrap[1]}': Dict[Item]}) return DictReplWithEsc(Str, NewDict, Esc) -def NumsFromFileName(Path): +def NumsFromFileName(Path:str): Name = Path.split('/')[-1] Split = len(Name) for i,e in enumerate(Name): @@ -126,7 +126,7 @@ def NumsFromFileName(Path): return Name[:i] return Path -def RevSort(List): +def RevSort(List:list): List.sort() List.reverse() return List @@ -157,7 +157,7 @@ def GetFullDate(Date): return None return datetime.strftime(datetime.strptime(Date, '%Y-%m-%d'), '%Y-%m-%dT%H:%M+00:00') -def LoadLocale(Lang): +def LoadLocale(Lang:str): Lang = Lang + '.json' Folder = f'{staticosoBaseDir()}Locale/' File = ReadFile(Folder + Lang) @@ -169,7 +169,10 @@ def LoadLocale(Lang): def IsLightRun(File, LimitFiles): return False if LimitFiles == False or File in LimitFiles else True -def PrintProcPercentDots(Proc, DivMult=1): +def NameSpace(From): + return SimpleNamespace(**From) + +def PrintProcPercentDots(Proc:dict, DivMult=1): Div = 5 * DivMult # 100/5 = 20 chars Num, Count = Proc['Num'], Proc['Count'] if int(((Num/Count)*100)/Div) != int((((Num+1)/Count)*100)/Div): @@ -177,27 +180,20 @@ def PrintProcPercentDots(Proc, DivMult=1): return True return False -def NameSpace(From): - return SimpleNamespace(**From) +def MultiProcFunctWrap(Args:dict): + PrintProcPercentDots(Args['Process']) + return Args['Process']['Funct'](Args) def DoMultiProc(Funct, ArgsCollection:list, Threads:int=cpu_count(), Progress:bool=False): - # The function should simply be like this - # def Funct(Args:dict): - # # Print the percentage dots, if needed - # PrintProcPercentDots(Args['Process']) - # # Call the real function that handles a single process at a time - # return Work(Args['a'], ...) - #if not Threads: - # Threads = cpu_count() FinalArgsCollection = [] for Index, Args in enumerate(ArgsCollection): FinalArgsCollection.append(Args) - FinalArgsCollection[Index].update({"Process": {"Num": Index, "Count": len(ArgsCollection)}}) + FinalArgsCollection[Index].update({"Process": {"Funct": Funct, "Num": Index, "Count": len(ArgsCollection)}}) Results = [] if Progress: os.system('printf "["') # Using system print because (see PrintProcPercentDots()) with Pool(Threads) as MultiprocPool: - Results = MultiprocPool.map(Funct, FinalArgsCollection) + Results = MultiprocPool.map(MultiProcFunctWrap if Progress else Funct, FinalArgsCollection) if Progress: os.system('printf "]\n"') # Newline after percentage dots return Results