From c4224e65a378fe66b79d1225bc3297c77088e673 Mon Sep 17 00:00:00 2001 From: octospacc Date: Mon, 29 Aug 2022 20:28:12 +0200 Subject: [PATCH] Misc. fixes/improvements; Fixed DiffBuild --- README.md | 4 +- Source/Build.py | 42 ++++++++----- Source/Modules/Gemini.py | 9 ++- Source/Modules/Site.py | 123 +++++++++++++++++++++++---------------- Source/Modules/Utils.py | 3 + TODO | 3 +- 6 files changed, 115 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index efb6ce4..ad753c0 100644 --- a/README.md +++ b/README.md @@ -71,11 +71,11 @@ Needed for Gemtext output support: - [x] Generation of global website menu as a tree or as a line - [x] Generation of page (titles) menu as a tree - [x] Auto-detection of titles in a page -- [x] _HTML_, TXT, _Extended Markdown_, and _Pug_ supported for input page files +- [x] **HTML**, **TXT**, **Extended Markdown**, and **Pug** supported as input page files - [ ] Out of heavy-WIP state ## Known issues (might need further investigation) - Bad HTML included in Markdown files can cause a build to fail entirely. - The program currently takes about 2 seconds to build a smallish site. While by itself that's not a long time, problems could arise for bigger sites. -- Ordering pages in the global menu with external configuration flags (outside the pages' source) yields broken and unpredictable results. \ No newline at end of file +- Ordering pages in the global menu with external configuration flags (outside the pages' source) yields broken and unpredictable results. diff --git a/Source/Build.py b/Source/Build.py index 1ef6bbe..be351e4 100755 --- a/Source/Build.py +++ b/Source/Build.py @@ -117,36 +117,44 @@ def Main(Args, FeedEntries): OutputDir = OptionChoose('public', Args.OutputDir, ReadConf(SiteConf, 'Site', 'OutputDir')) OutputDir = OutputDir.removesuffix('/') CheckSafeOutputDir(OutputDir) - print(f"[I] Outputting to {OutputDir}/") + print(f"[I] Outputting to: {OutputDir}/") DiffBuild = Args.DiffBuild + BlogName = OptionChoose('', Args.BlogName, ReadConf(SiteConf, 'Site', 'BlogName')) SiteTagline = OptionChoose('', Args.SiteTagline, ReadConf(SiteConf, 'Site', 'Tagline')) SiteTemplate = OptionChoose('Default.html', Args.SiteTemplate, ReadConf(SiteConf, 'Site', 'Template')) SiteDomain = OptionChoose('', Args.SiteDomain, ReadConf(SiteConf, 'Site', 'Domain')) - SiteDomain = SiteDomain.removesuffix('/') SiteRoot = OptionChoose('/', Args.SiteRoot, ReadConf(SiteConf, 'Site', 'Root')) SiteLang = OptionChoose('en', Args.SiteLang, ReadConf(SiteConf, 'Site', 'Lang')) - Locale = LoadLocale(SiteLang) - MastodonURL = OptionChoose('', Args.MastodonURL, ReadConf(SiteConf, 'Mastodon', 'URL')) - MastodonToken = OptionChoose('', Args.MastodonToken, ReadConf(SiteConf, 'Mastodon', 'Token')) + + Sorting = literal_eval(OptionChoose('{}', Args.Sorting, ReadConf(SiteConf, 'Site', 'Sorting'))) DynamicParts = literal_eval(OptionChoose('{}', Args.DynamicParts, ReadConf(SiteConf, 'Site', 'DynamicParts'))) - MarkdownExts = literal_eval(OptionChoose(str(MarkdownExtsDefault), Args.MarkdownExts, ReadConf(SiteConf, 'Markdown', 'Exts'))) + NoScripts = StringBoolChoose(False, Args.NoScripts, ReadConf(SiteConf, 'Site', 'NoScripts')) + ActivityPubTypeFilter = OptionChoose('Post', Args.ActivityPubTypeFilter, ReadConf(SiteConf, 'ActivityPub', 'TypeFilter')) ActivityPubHoursLimit = OptionChoose(168, Args.ActivityPubHoursLimit, ReadConf(SiteConf, 'ActivityPub', 'HoursLimit')) - FeedCategoryFilter = OptionChoose('Blog', Args.FeedCategoryFilter, ReadConf(SiteConf, 'Feed', 'CategoryFilter')) + + MastodonURL = OptionChoose('', Args.MastodonURL, ReadConf(SiteConf, 'Mastodon', 'URL')) + MastodonToken = OptionChoose('', Args.MastodonToken, ReadConf(SiteConf, 'Mastodon', 'Token')) + + MarkdownExts = literal_eval(OptionChoose(str(MarkdownExtsDefault), Args.MarkdownExts, ReadConf(SiteConf, 'Markdown', 'Exts'))) + SitemapOutput = StringBoolChoose(True, Args.SitemapOutput, ReadConf(SiteConf, 'Sitemap', 'Output')) + Minify = StringBoolChoose(False, Args.Minify, ReadConf(SiteConf, 'Minify', 'Minify')) MinifyKeepComments = StringBoolChoose(False, Args.MinifyKeepComments, ReadConf(SiteConf, 'Minify', 'KeepComments')) - NoScripts = StringBoolChoose(False, Args.NoScripts, ReadConf(SiteConf, 'Site', 'NoScripts')) + ImgAltToTitle = StringBoolChoose(True, Args.ImgAltToTitle, ReadConf(SiteConf, 'Site', 'ImgAltToTitle')) ImgTitleToAlt = StringBoolChoose(False, Args.ImgTitleToAlt, ReadConf(SiteConf, 'Site', 'ImgTitleToAlt')) + CategoriesAutomatic = StringBoolChoose(False, Args.CategoriesAutomatic, ReadConf(SiteConf, 'Categories', 'Automatic')) CategoriesUncategorized = OptionChoose('Uncategorized', Args.CategoriesUncategorized, ReadConf(SiteConf, 'Categories', 'Uncategorized')) + GemtextOutput = StringBoolChoose(False, Args.GemtextOutput, ReadConf(SiteConf, 'Gemtext', 'Output')) GemtextHeader = Args.GemtextHeader if Args.GemtextHeader else ReadConf(SiteConf, 'Gemtext', 'Header') if ReadConf(SiteConf, 'Gemtext', 'Header') else f"# {SiteName}\n\n" if SiteName else '' - SitemapOutput = StringBoolChoose(True, Args.SitemapOutput, ReadConf(SiteConf, 'Sitemap', 'Output')) - FeedEntries = int(FeedEntries) if (FeedEntries or FeedEntries == 0) and FeedEntries != 'Default' else int(ReadConf(SiteConf, 'Site', 'FeedEntries')) if ReadConf(SiteConf, 'Site', 'FeedEntries') else 10 - Sorting = literal_eval(OptionChoose('{}', Args.Sorting, ReadConf(SiteConf, 'Site', 'Sorting'))) + + FeedCategoryFilter = OptionChoose('Blog', Args.FeedCategoryFilter, ReadConf(SiteConf, 'Feed', 'CategoryFilter')) + FeedEntries = int(FeedEntries) if (FeedEntries or FeedEntries == 0) and FeedEntries != 'Default' else int(ReadConf(SiteConf, 'Feed', 'Entries')) if ReadConf(SiteConf, 'Feed', 'Entries') else 10 MenuEntries = ReadConf(SiteConf, 'Menu') if MenuEntries: @@ -154,6 +162,9 @@ def Main(Args, FeedEntries): else: ConfMenu = [] + SiteDomain = SiteDomain.removesuffix('/') + Locale = LoadLocale(SiteLang) + if DiffBuild: print("[I] Build mode: Differential") LimitFiles = GetModifiedFiles(OutputDir) @@ -202,7 +213,8 @@ def Main(Args, FeedEntries): ImgAltToTitle=ImgAltToTitle, ImgTitleToAlt=ImgTitleToAlt, Sorting=SetSorting(Sorting), MarkdownExts=MarkdownExts, - AutoCategories=CategoriesAutomatic) + AutoCategories=CategoriesAutomatic, + CategoryUncategorized=CategoriesUncategorized) if FeedEntries != 0: print("[I] Generating Feeds") @@ -235,6 +247,8 @@ def Main(Args, FeedEntries): MastodonPosts = [] for File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image in Pages: + if IsLightRun(File, LimitFiles): + continue File = f"{OutputDir}/{StripExt(File)}.html" Content = ReadFile(File) Post = '' @@ -250,7 +264,7 @@ def Main(Args, FeedEntries): if GemtextOutput: print("[I] Generating Gemtext") - GemtextCompileList(OutputDir, Pages, GemtextHeader) + GemtextCompileList(OutputDir, Pages, LimitFiles, GemtextHeader) print("[I] Cleaning Temporary Files") DelTmp(OutputDir) @@ -292,8 +306,8 @@ if __name__ == '__main__': Parser.add_argument('--FeedCategoryFilter', type=str) Parser.add_argument('--ActivityPubTypeFilter', type=str, help=argparse.SUPPRESS) Parser.add_argument('--ActivityPubHoursLimit', type=int) - Parser.add_argument('--CategoriesUncategorized', type=str) Parser.add_argument('--CategoriesAutomatic', type=str) + Parser.add_argument('--CategoriesUncategorized', type=str) Args = Parser.parse_args() try: diff --git a/Source/Modules/Gemini.py b/Source/Modules/Gemini.py index 0fae65f..518ee0a 100644 --- a/Source/Modules/Gemini.py +++ b/Source/Modules/Gemini.py @@ -22,9 +22,11 @@ def FixGemlogDateLine(Line): Line = Words[0] + '\n' + Words[1][1:] + ' ' + ' '.join(Words[2:]) return Line -def GemtextCompileList(OutputDir, Pages, Header=''): +def GemtextCompileList(OutputDir, Pages, LimitFiles, Header=''): Cmd = '' for File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image in Pages: + if IsLightRun(File, LimitFiles): + continue Src = f"{OutputDir}.gmi/{StripExt(File)}.html.tmp" Dst = f"{OutputDir}.gmi/{StripExt(File)}.gmi" SlimHTML = StripAttrs(SlimHTML) @@ -33,8 +35,11 @@ def GemtextCompileList(OutputDir, Pages, Header=''): SlimHTML = SlimHTML.replace(j, '') WriteFile(Src, SlimHTML.replace('', '
').replace('.html', '.gmi')) # TODO: Adjust links properly.. Cmd += f'cat "{Src}" | html2gmi > "{Dst}"; ' - os.system(Cmd) + if Cmd: + os.system(Cmd) for File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image in Pages: + if IsLightRun(File, LimitFiles): + continue Dst = f"{OutputDir}.gmi/{StripExt(File)}.gmi" Gemtext = '' for Line in ReadFile(Dst).splitlines(): diff --git a/Source/Modules/Site.py b/Source/Modules/Site.py index 2ff32f4..db7bba7 100644 --- a/Source/Modules/Site.py +++ b/Source/Modules/Site.py @@ -149,7 +149,7 @@ def TemplatePreprocessor(Text): Meta.update({i:MetaDefault[i]}) return Meta -def PagePreprocessor(Path, Type, SiteTemplate, SiteRoot, GlobalMacros, LightRun=False): +def PagePreprocessor(Path, Type, SiteTemplate, SiteRoot, GlobalMacros, CategoryUncategorized, LightRun=False): File = ReadFile(Path) Path = Path.lower() Content, Titles, DashyTitles, HTMLTitlesFound, Macros, Meta, MetaDefault = '', [], [], False, '', '', { @@ -236,7 +236,7 @@ def PagePreprocessor(Path, Type, SiteTemplate, SiteRoot, GlobalMacros, LightRun= Meta.update({i:MetaDefault[i]}) if Meta['Index'] in ('Default', 'Unspecified'): if not Meta['Categories']: - Meta['Categories'] = ['Uncategorized'] + Meta['Categories'] = [CategoryUncategorized] if Meta['Type'] == 'Page': Meta['Index'] = 'False' elif Meta['Type'] == 'Post': @@ -302,7 +302,7 @@ def CanIndex(Index, For): else: return True if Index == For else False -def PatchHTML(File, HTML, StaticPartsText, DynamicParts, DynamicPartsText, HTMLPagesList, PagePath, Content, Titles, Meta, SiteRoot, SiteName, BlogName, FolderRoots, Categories, SiteLang, Locale): +def PatchHTML(File, HTML, StaticPartsText, DynamicParts, DynamicPartsText, HTMLPagesList, PagePath, Content, Titles, Meta, SiteRoot, SiteName, BlogName, FolderRoots, Categories, SiteLang, Locale, LightRun): HTMLTitles = FormatTitles(Titles) BodyDescription, BodyImage = '', '' if not File.lower().endswith('.txt'): @@ -339,35 +339,42 @@ def PatchHTML(File, HTML, StaticPartsText, DynamicParts, DynamicPartsText, HTMLP for e in StaticPartsText: HTML = ReplWithEsc(HTML, f"[staticoso:StaticPart:{e}]", StaticPartsText[e]) - HTML = DictReplWithEsc( - HTML, { - '[staticoso:Site:Menu]': HTMLPagesList, - '[staticoso:Page:Lang]': SiteLang, - '[staticoso:Page:Chapters]': HTMLTitles, - '[staticoso:Page:Title]': Title, - '[staticoso:Page:Description]': Description, - '[staticoso:Page:Image]': Image, - '[staticoso:Page:Path]': PagePath, - '[staticoso:Page:Style]': Meta['Style'], - '[staticoso:Page:Content]': Content, - '[staticoso:Page:ContentInfo]': MakeContentHeader(Meta, Locale, MakeCategoryLine(File, Meta)), - '[staticoso:BuildTime]': datetime.now().strftime('%Y-%m-%d %H:%M'), - '[staticoso:Site:Name]': SiteName, - '[staticoso:Site:AbsoluteRoot]': SiteRoot, - '[staticoso:Site:RelativeRoot]': GetPathLevels(PagePath) - }) - for e in Meta['Macros']: - HTML = ReplWithEsc(HTML, f"[:{e}:]", Meta['Macros'][e]) - for e in FolderRoots: - HTML = ReplWithEsc(HTML, f"[staticoso:Folder:{e}:AbsoluteRoot]", FolderRoots[e]) - for e in Categories: - HTML = ReplWithEsc(HTML, f"[staticoso:Category:{e}]", Categories[e]) - HTML = ReplWithEsc(HTML, f"[staticoso:Category:{e}]", Categories[e]) + + if LightRun: + HTML = None + else: + HTML = DictReplWithEsc( + HTML, { + '[staticoso:Site:Menu]': HTMLPagesList, + '[staticoso:Page:Lang]': SiteLang, + '[staticoso:Page:Chapters]': HTMLTitles, + '[staticoso:Page:Title]': Title, + '[staticoso:Page:Description]': Description, + '[staticoso:Page:Image]': Image, + '[staticoso:Page:Path]': PagePath, + '[staticoso:Page:Style]': Meta['Style'], + '[staticoso:Page:Content]': Content, + '[staticoso:Page:ContentInfo]': MakeContentHeader(Meta, Locale, MakeCategoryLine(File, Meta)), + '[staticoso:BuildTime]': datetime.now().strftime('%Y-%m-%d %H:%M'), + '[staticoso:Site:Name]': SiteName, + '[staticoso:Site:AbsoluteRoot]': SiteRoot, + '[staticoso:Site:RelativeRoot]': GetPathLevels(PagePath) + }) + for e in Meta['Macros']: + HTML = ReplWithEsc(HTML, f"[:{e}:]", Meta['Macros'][e]) + for e in FolderRoots: + HTML = ReplWithEsc(HTML, f"[staticoso:Folder:{e}:AbsoluteRoot]", FolderRoots[e]) + for e in Categories: + HTML = ReplWithEsc(HTML, f"[staticoso:Category:{e}]", Categories[e]) + HTML = ReplWithEsc(HTML, f"[staticoso:Category:{e}]", Categories[e]) # TODO: Clean this doubling? ContentHTML = Content ContentHTML = DictReplWithEsc( ContentHTML, { + '[staticoso:Page:Title]': Title, + '[staticoso:Page:Description]': Description, + '[staticoso:Site:Name]': SiteName, '[staticoso:Site:AbsoluteRoot]': SiteRoot, '[staticoso:Site:RelativeRoot]': GetPathLevels(PagePath) }) @@ -378,11 +385,10 @@ def PatchHTML(File, HTML, StaticPartsText, DynamicParts, DynamicPartsText, HTMLP for e in Categories: ContentHTML = ReplWithEsc(ContentHTML, f"[staticoso:Category:{e}]", Categories[e]) ContentHTML = ReplWithEsc(ContentHTML, f"[staticoso:Category:{e}]", Categories[e]) - SlimHTML = HTMLPagesList + ContentHTML - return HTML, ContentHTML, SlimHTML, Description, Image + return HTML, ContentHTML, Description, Image -def MakeSite(OutputDir, LimitFiles, TemplatesText, StaticPartsText, DynamicParts, DynamicPartsText, ConfMenu, GlobalMacros, SiteName, BlogName, SiteTagline, SiteTemplate, SiteDomain, SiteRoot, FolderRoots, SiteLang, Locale, Minify, MinifyKeepComments, NoScripts, ImgAltToTitle, ImgTitleToAlt, Sorting, MarkdownExts, AutoCategories): +def MakeSite(OutputDir, LimitFiles, TemplatesText, StaticPartsText, DynamicParts, DynamicPartsText, ConfMenu, GlobalMacros, SiteName, BlogName, SiteTagline, SiteTemplate, SiteDomain, SiteRoot, FolderRoots, SiteLang, Locale, Minify, MinifyKeepComments, NoScripts, ImgAltToTitle, ImgTitleToAlt, Sorting, MarkdownExts, AutoCategories, CategoryUncategorized): PagesPaths, PostsPaths, Pages, MadePages, Categories = [], [], [], [], {} for Ext in FileExtensions['Pages']: for File in Path('Pages').rglob(f"*.{Ext}"): @@ -408,7 +414,7 @@ def MakeSite(OutputDir, LimitFiles, TemplatesText, StaticPartsText, DynamicParts for File in Files: TempPath = f"{PathPrefix}{File}" LightRun = False if LimitFiles == False or TempPath in LimitFiles else True - Content, Titles, Meta = PagePreprocessor(f"{Type}s/{File}", Type, SiteTemplate, SiteRoot, GlobalMacros, LightRun=LightRun) + Content, Titles, Meta = PagePreprocessor(f"{Type}s/{File}", Type, SiteTemplate, SiteRoot, GlobalMacros, CategoryUncategorized, LightRun=LightRun) Pages += [[TempPath, Content, Titles, Meta]] for Cat in Meta['Categories']: Categories.update({Cat:''}) @@ -440,7 +446,7 @@ def MakeSite(OutputDir, LimitFiles, TemplatesText, StaticPartsText, DynamicParts File = f"Categories/{Cat}.md" FilePath = f"{OutputDir}/{File}" WriteFile(FilePath, CategoryPageTemplate.format(Title=Cat)) - Content, Titles, Meta = PagePreprocessor(FilePath, SiteRoot) + Content, Titles, Meta = PagePreprocessor(FilePath, 'Page', SiteTemplate, SiteRoot, GlobalMacros, CategoryUncategorized, LightRun=LightRun) Pages += [[File, Content, Titles, Meta]] for i,e in enumerate(ConfMenu): @@ -451,6 +457,8 @@ def MakeSite(OutputDir, LimitFiles, TemplatesText, StaticPartsText, DynamicParts print("[I] Writing Pages") for File, Content, Titles, Meta in Pages: + LightRun = False if LimitFiles == False or File in LimitFiles else True + PagePath = f"{OutputDir}/{StripExt(File)}.html" if File.lower().endswith(FileExtensions['Markdown']): Content = markdown(PagePostprocessor('md', Content, Meta), extensions=MarkdownExts) @@ -461,19 +469,22 @@ def MakeSite(OutputDir, LimitFiles, TemplatesText, StaticPartsText, DynamicParts elif File.lower().endswith(FileExtensions['HTML']): Content = ReadFile(PagePath) - TemplateMeta = TemplatePreprocessor(TemplatesText[Meta['Template']]) - HTMLPagesList = GetHTMLPagesList( - Pages=Pages, - BlogName=BlogName, - SiteRoot=SiteRoot, - PathPrefix=GetPathLevels(File), - Unite=ConfMenu, - Type='Page', - For='Menu', - MarkdownExts=MarkdownExts, - MenuStyle=TemplateMeta['MenuStyle']) + if LightRun: + HTMLPagesList = None + else: + TemplateMeta = TemplatePreprocessor(TemplatesText[Meta['Template']]) + HTMLPagesList = GetHTMLPagesList( + Pages=Pages, + BlogName=BlogName, + SiteRoot=SiteRoot, + PathPrefix=GetPathLevels(File), + Unite=ConfMenu, + Type='Page', + For='Menu', + MarkdownExts=MarkdownExts, + MenuStyle=TemplateMeta['MenuStyle']) - HTML, ContentHTML, SlimHTML, Description, Image = PatchHTML( + HTML, ContentHTML, Description, Image = PatchHTML( File=File, HTML=TemplatesText[Meta['Template']], StaticPartsText=StaticPartsText, @@ -490,16 +501,30 @@ def MakeSite(OutputDir, LimitFiles, TemplatesText, StaticPartsText, DynamicParts FolderRoots=FolderRoots, Categories=Categories, SiteLang=SiteLang, - Locale=Locale) + Locale=Locale, + LightRun=LightRun) if Minify: - HTML = DoMinifyHTML(HTML, MinifyKeepComments) + if not LightRun: + HTML = DoMinifyHTML(HTML, MinifyKeepComments) + ContentHTML = DoMinifyHTML(ContentHTML, MinifyKeepComments) if NoScripts: - HTML = StripTags(HTML, ['script']) + if not LightRun: + HTML = StripTags(HTML, ['script']) + ContentHTML = StripTags(ContentHTML, ['script']) if ImgAltToTitle or ImgTitleToAlt: - HTML = WriteImgAltAndTitle(HTML, ImgAltToTitle, ImgTitleToAlt) + if not LightRun: + HTML = WriteImgAltAndTitle(HTML, ImgAltToTitle, ImgTitleToAlt) + ContentHTML = WriteImgAltAndTitle(ContentHTML, ImgAltToTitle, ImgTitleToAlt) + + if LightRun: + SlimHTML = None + else: + SlimHTML = HTMLPagesList + ContentHTML + + if not LightRun: + WriteFile(PagePath, HTML) - WriteFile(PagePath, HTML) MadePages += [[File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image]] return MadePages diff --git a/Source/Modules/Utils.py b/Source/Modules/Utils.py index 8c067f3..5bb9de2 100644 --- a/Source/Modules/Utils.py +++ b/Source/Modules/Utils.py @@ -154,3 +154,6 @@ def LoadLocale(Lang): return json.loads(File) else: return json.loads(ReadFile(Folder + 'en.json')) + +def IsLightRun(File, LimitFiles): + return False if LimitFiles == False or File in LimitFiles else True diff --git a/TODO b/TODO index 10ad82d..5e72798 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,8 @@ - Check if external tools (pug-cli, html2gmi) are installed - Fix and optimize differential building - Static code syntax highlighing -- Override internal HTML snippets (with config file in Templates/NAME.ini) +- Override internal HTML snippets (meta lines, page lists, ...) with config file in Templates/NAME.ini - Specify input folder(s) -- Customize HTML source code for Meta lines / page lists - Show page size/words/time in meta line - Add feed support for diary-like pages - Fix excess whitespace in some section/menu titles