From a1e77b8f9156f4190ea26f7d34fa67a11f916626 Mon Sep 17 00:00:00 2001 From: octospacc Date: Mon, 29 Aug 2022 17:50:14 +0200 Subject: [PATCH] More coherent conf. flags, Updated README, minor code improv. --- README.md | 19 +++++++--- Source/Build.py | 31 +++++++++------- Source/Modules/Config.py | 4 +-- Source/Modules/HTML.py | 13 +++++++ Source/Modules/Site.py | 76 +++++++++++++++++++--------------------- Source/Modules/Utils.py | 6 ++++ 6 files changed, 90 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index cd326b8..efb6ce4 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,13 @@ Obviously, it's built with staticoso itself 😁️. Its source repo can be foun Keep in mind that, currently, it's still very incomplete. **Any help**, from writing the documentation to creating a decent HTML+CSS template for its site, **is more than welcome**. ## Dependencies -- [Python == 3.10.4](https://python.org) + +- [Python >= 3.10](https://python.org) - (Included) [Python Markdown == 3.3.7](https://pypi.org/project/Markdown) - (Included) Third-Party Extensions for Python Markdown: [markdown_del_ins](https://github.com/honzajavorek/markdown-del-ins), [mdx_subscript](https://github.com/jambonrose/markdown_subscript_extension), [mdx_superscript](https://github.com/jambonrose/markdown_superscript_extension) - (Included) [Beautiful Soup == 4.11.1](https://pypi.org/project/beautifulsoup4) - (Included) [feedgen == 0.9.0](https://pypi.org/project/feedgen) + - [lxml == 4.9.1](https://pypi.org/project/lxml) - (Included) [htmlmin == 0.1.12](https://pypi.org/project/htmlmin) ### Optional dependencies @@ -29,7 +31,7 @@ Keep in mind that, currently, it's still very incomplete. **Any help**, from wri Needed for Pug input support: - [node == 12.22.5](https://nodejs.org) - [npm == 7.5.2](https://www.npmjs.com) -- (Included) [pug-cli == 1.0.0-alpha6](https://npmjs.com/package/pug-cli) +- (Included, must be manually installed in system $PATH) [pug-cli == 1.0.0-alpha6](https://npmjs.com/package/pug-cli) Needed for Gemtext output support: @@ -37,14 +39,17 @@ Needed for Gemtext output support: - [html2gmi](https://github.com/LukeEmmet/html2gmi) ## Features roadmap + +- [ ] Overriding internal HTML snippets for template-specific ones +- [ ] Static syntax highlighing for code blocks in any page - [x] File name used as a title for pages without one - [ ] Custom category names in header links - [ ] Choosing custom name for Blog and Uncategorized categories - [ ] Choosing to use a template for all pages in a folder -- [ ] Configuration with both INI files and CLI arguments +- [x] Configuration with both INI files and CLI arguments - [ ] Category-based feeds - [ ] Support for multi-language sites -- [x] The `title` attribute is added to images which only have `alt` (for desktop accessibility) +- [x] The `title` attribute is added to images which only have `alt` (for desktop accessibility), or viceversa - [x] Local (per-page) and global (per-site) macros - [x] ActivityPub (Mastodon) support (Feed + embedded comments) - [ ] Polished Gemtext generation @@ -68,3 +73,9 @@ Needed for Gemtext output support: - [x] Auto-detection of titles in a page - [x] _HTML_, TXT, _Extended Markdown_, and _Pug_ supported for input page files - [ ] Out of heavy-WIP state + +## Known issues (might need further investigation) + +- Bad HTML included in Markdown files can cause a build to fail entirely. +- The program currently takes about 2 seconds to build a smallish site. While by itself that's not a long time, problems could arise for bigger sites. +- Ordering pages in the global menu with external configuration flags (outside the pages' source) yields broken and unpredictable results. \ No newline at end of file diff --git a/Source/Build.py b/Source/Build.py index 5a99d38..1ef6bbe 100755 --- a/Source/Build.py +++ b/Source/Build.py @@ -135,14 +135,16 @@ def Main(Args, FeedEntries): ActivityPubTypeFilter = OptionChoose('Post', Args.ActivityPubTypeFilter, ReadConf(SiteConf, 'ActivityPub', 'TypeFilter')) ActivityPubHoursLimit = OptionChoose(168, Args.ActivityPubHoursLimit, ReadConf(SiteConf, 'ActivityPub', 'HoursLimit')) FeedCategoryFilter = OptionChoose('Blog', Args.FeedCategoryFilter, ReadConf(SiteConf, 'Feed', 'CategoryFilter')) - Minify = StringBoolChoose(False, Args.Minify, ReadConf(SiteConf, 'Site', 'Minify')) + Minify = StringBoolChoose(False, Args.Minify, ReadConf(SiteConf, 'Minify', 'Minify')) + MinifyKeepComments = StringBoolChoose(False, Args.MinifyKeepComments, ReadConf(SiteConf, 'Minify', 'KeepComments')) NoScripts = StringBoolChoose(False, Args.NoScripts, ReadConf(SiteConf, 'Site', 'NoScripts')) ImgAltToTitle = StringBoolChoose(True, Args.ImgAltToTitle, ReadConf(SiteConf, 'Site', 'ImgAltToTitle')) ImgTitleToAlt = StringBoolChoose(False, Args.ImgTitleToAlt, ReadConf(SiteConf, 'Site', 'ImgTitleToAlt')) - AutoCategories = StringBoolChoose(False, Args.AutoCategories, ReadConf(SiteConf, 'Site', 'AutoCategories')) - GemtextOut = StringBoolChoose(False, Args.GemtextOut, ReadConf(SiteConf, 'Site', 'GemtextOut')) + CategoriesAutomatic = StringBoolChoose(False, Args.CategoriesAutomatic, ReadConf(SiteConf, 'Categories', 'Automatic')) + CategoriesUncategorized = OptionChoose('Uncategorized', Args.CategoriesUncategorized, ReadConf(SiteConf, 'Categories', 'Uncategorized')) + GemtextOutput = StringBoolChoose(False, Args.GemtextOutput, ReadConf(SiteConf, 'Gemtext', 'Output')) GemtextHeader = Args.GemtextHeader if Args.GemtextHeader else ReadConf(SiteConf, 'Gemtext', 'Header') if ReadConf(SiteConf, 'Gemtext', 'Header') else f"# {SiteName}\n\n" if SiteName else '' - SitemapOut = StringBoolChoose(True, Args.SitemapOut, ReadConf(SiteConf, 'Site', 'SitemapOut')) + SitemapOutput = StringBoolChoose(True, Args.SitemapOutput, ReadConf(SiteConf, 'Sitemap', 'Output')) FeedEntries = int(FeedEntries) if (FeedEntries or FeedEntries == 0) and FeedEntries != 'Default' else int(ReadConf(SiteConf, 'Site', 'FeedEntries')) if ReadConf(SiteConf, 'Site', 'FeedEntries') else 10 Sorting = literal_eval(OptionChoose('{}', Args.Sorting, ReadConf(SiteConf, 'Site', 'Sorting'))) @@ -163,12 +165,12 @@ def Main(Args, FeedEntries): if os.path.isdir('Pages'): HavePages = True shutil.copytree('Pages', OutputDir, dirs_exist_ok=True) - if GemtextOut: + if GemtextOutput: shutil.copytree('Pages', f"{OutputDir}.gmi", ignore=IgnoreFiles, dirs_exist_ok=True) if os.path.isdir('Posts'): HavePosts = True shutil.copytree('Posts', f"{OutputDir}/Posts", dirs_exist_ok=True) - if GemtextOut: + if GemtextOutput: shutil.copytree('Posts', f"{OutputDir}.gmi/Posts", ignore=IgnoreFiles, dirs_exist_ok=True) if not (HavePages or HavePosts): @@ -195,11 +197,12 @@ def Main(Args, FeedEntries): SiteLang=SiteLang, Locale=Locale, Minify=Minify, + MinifyKeepComments=MinifyKeepComments, NoScripts=NoScripts, ImgAltToTitle=ImgAltToTitle, ImgTitleToAlt=ImgTitleToAlt, Sorting=SetSorting(Sorting), MarkdownExts=MarkdownExts, - AutoCategories=AutoCategories) + AutoCategories=CategoriesAutomatic) if FeedEntries != 0: print("[I] Generating Feeds") @@ -245,14 +248,14 @@ def Main(Args, FeedEntries): Content = ReplWithEsc(Content, '[staticoso:Comments]', Post) WriteFile(File, Content) - if GemtextOut: + if GemtextOutput: print("[I] Generating Gemtext") GemtextCompileList(OutputDir, Pages, GemtextHeader) print("[I] Cleaning Temporary Files") DelTmp(OutputDir) - if SitemapOut: + if SitemapOutput: print("[I] Generating Sitemap") MakeSitemap(OutputDir, Pages, SiteDomain) @@ -261,7 +264,7 @@ def Main(Args, FeedEntries): if __name__ == '__main__': Parser = argparse.ArgumentParser() - Parser.add_argument('--DiffBuild', action='store_true') + Parser.add_argument('--DiffBuild', type=str) Parser.add_argument('--OutputDir', type=str) #Parser.add_argument('--InputDir', type=str) Parser.add_argument('--Sorting', type=str) @@ -272,13 +275,14 @@ if __name__ == '__main__': Parser.add_argument('--SiteTemplate', type=str) Parser.add_argument('--SiteDomain', type=str) Parser.add_argument('--Minify', type=str) + Parser.add_argument('--MinifyKeepComments', type=str) Parser.add_argument('--NoScripts', type=str) Parser.add_argument('--ImgAltToTitle', type=str) Parser.add_argument('--ImgTitleToAlt', type=str) - Parser.add_argument('--GemtextOut', type=str) + Parser.add_argument('--GemtextOutput', type=str) Parser.add_argument('--GemtextHeader', type=str) Parser.add_argument('--SiteTagline', type=str) - Parser.add_argument('--SitemapOut', type=str) + Parser.add_argument('--SitemapOutput', type=str) Parser.add_argument('--FeedEntries', type=str) Parser.add_argument('--FolderRoots', type=str) Parser.add_argument('--DynamicParts', type=str) @@ -288,7 +292,8 @@ if __name__ == '__main__': Parser.add_argument('--FeedCategoryFilter', type=str) Parser.add_argument('--ActivityPubTypeFilter', type=str, help=argparse.SUPPRESS) Parser.add_argument('--ActivityPubHoursLimit', type=int) - Parser.add_argument('--AutoCategories', type=str) + Parser.add_argument('--CategoriesUncategorized', type=str) + Parser.add_argument('--CategoriesAutomatic', type=str) Args = Parser.parse_args() try: diff --git a/Source/Modules/Config.py b/Source/Modules/Config.py index 984a9b2..4a054da 100644 --- a/Source/Modules/Config.py +++ b/Source/Modules/Config.py @@ -37,8 +37,8 @@ def EvalOpt(Opt): else: return None -def OptionChoose(Default, Primary, Secondary): - return Primary if Primary != None else Secondary if Secondary != None else Default +def OptionChoose(Default, Primary, Secondary, Tertiary=None): + return Primary if Primary != None else Secondary if Secondary != None else Tertiary if Tertiary != None else Default def StringBoolChoose(Default, Primary, Secondary): Var = Default diff --git a/Source/Modules/HTML.py b/Source/Modules/HTML.py index e76d530..1589b0b 100644 --- a/Source/Modules/HTML.py +++ b/Source/Modules/HTML.py @@ -9,6 +9,7 @@ import html import warnings +from Libs import htmlmin from Libs.bs4 import BeautifulSoup from Modules.Utils import * @@ -70,3 +71,15 @@ def SquareFnrefs(HTML): # Different combinations of formatting for Soup .prettif s = t.find('a') s.replace_with(f'[{t}]') return str(Soup.prettify(formatter=None)) + +def DoMinifyHTML(HTML, KeepComments): + return htmlmin.minify( + input=HTML, + remove_comments=not KeepComments, + remove_empty_space=True, + remove_all_empty_space=False, + reduce_empty_attributes=True, + reduce_boolean_attributes=True, + remove_optional_attribute_quotes=True, + convert_charrefs=True, + keep_pre=True) diff --git a/Source/Modules/Site.py b/Source/Modules/Site.py index e89e3d6..2ff32f4 100644 --- a/Source/Modules/Site.py +++ b/Source/Modules/Site.py @@ -8,7 +8,6 @@ | ================================= """ from datetime import datetime -from Libs import htmlmin from Libs.bs4 import BeautifulSoup from Modules.Config import * from Modules.HTML import * @@ -18,6 +17,15 @@ from Modules.Utils import * HTMLSectionTitleLine = '» {Title}' #PugSectionTitleLine = "{Line[:Index]}{Line[Index:Index+2]}.SectionHeading #[span.SectionLink #[a(href='#{DashTitle}') #[span »]] ]#[span#{DashTitle}.SectionTitle {Line[Index+2:]}]" +CategoryPageTemplate = """\ +// Title: {Name} +// Type: Page +// Index: True + +# {Name} + +
[staticoso:Category:{Name}]
+""" def DashifyTitle(Title, Done=[]): return UndupeStr(DashifyStr(Title.lstrip(' ').rstrip(' ')), Done, '-') @@ -331,54 +339,50 @@ def PatchHTML(File, HTML, StaticPartsText, DynamicParts, DynamicPartsText, HTMLP for e in StaticPartsText: HTML = ReplWithEsc(HTML, f"[staticoso:StaticPart:{e}]", StaticPartsText[e]) - HTML = ReplWithEsc(HTML, '[staticoso:Site:Menu]', HTMLPagesList) - HTML = ReplWithEsc(HTML, '[staticoso:Page:Lang]', SiteLang) - HTML = ReplWithEsc(HTML, '[staticoso:Page:Chapters]', HTMLTitles) - HTML = ReplWithEsc(HTML, '[staticoso:Page:Title]', Title) - HTML = ReplWithEsc(HTML, '[staticoso:Page:Description]', Description) - HTML = ReplWithEsc(HTML, '[staticoso:Page:Image]', Image) - HTML = ReplWithEsc(HTML, '[staticoso:Page:Path]', PagePath) - HTML = ReplWithEsc(HTML, '[staticoso:Page:Style]', Meta['Style']) - HTML = ReplWithEsc(HTML, '[staticoso:Page:Content]', Content) - HTML = ReplWithEsc(HTML, '[staticoso:Page:ContentInfo]', MakeContentHeader(Meta, Locale, MakeCategoryLine(File, Meta))) - HTML = ReplWithEsc(HTML, '[staticoso:BuildTime]', datetime.now().strftime('%Y-%m-%d %H:%M')) - HTML = ReplWithEsc(HTML, '[staticoso:Site:Name]', SiteName) - HTML = ReplWithEsc(HTML, '[staticoso:Site:AbsoluteRoot]', SiteRoot) - HTML = ReplWithEsc(HTML, '[staticoso:Site:RelativeRoot]', GetPathLevels(PagePath)) + HTML = DictReplWithEsc( + HTML, { + '[staticoso:Site:Menu]': HTMLPagesList, + '[staticoso:Page:Lang]': SiteLang, + '[staticoso:Page:Chapters]': HTMLTitles, + '[staticoso:Page:Title]': Title, + '[staticoso:Page:Description]': Description, + '[staticoso:Page:Image]': Image, + '[staticoso:Page:Path]': PagePath, + '[staticoso:Page:Style]': Meta['Style'], + '[staticoso:Page:Content]': Content, + '[staticoso:Page:ContentInfo]': MakeContentHeader(Meta, Locale, MakeCategoryLine(File, Meta)), + '[staticoso:BuildTime]': datetime.now().strftime('%Y-%m-%d %H:%M'), + '[staticoso:Site:Name]': SiteName, + '[staticoso:Site:AbsoluteRoot]': SiteRoot, + '[staticoso:Site:RelativeRoot]': GetPathLevels(PagePath) + }) for e in Meta['Macros']: HTML = ReplWithEsc(HTML, f"[:{e}:]", Meta['Macros'][e]) for e in FolderRoots: HTML = ReplWithEsc(HTML, f"[staticoso:Folder:{e}:AbsoluteRoot]", FolderRoots[e]) for e in Categories: HTML = ReplWithEsc(HTML, f"[staticoso:Category:{e}]", Categories[e]) + HTML = ReplWithEsc(HTML, f"[staticoso:Category:{e}]", Categories[e]) # TODO: Clean this doubling? ContentHTML = Content - ContentHTML = ReplWithEsc(ContentHTML, '[staticoso:Site:AbsoluteRoot]', SiteRoot) - ContentHTML = ReplWithEsc(ContentHTML, '[staticoso:Site:RelativeRoot]', GetPathLevels(PagePath)) + ContentHTML = DictReplWithEsc( + ContentHTML, { + '[staticoso:Site:AbsoluteRoot]': SiteRoot, + '[staticoso:Site:RelativeRoot]': GetPathLevels(PagePath) + }) for e in Meta['Macros']: ContentHTML = ReplWithEsc(ContentHTML, f"[:{e}:]", Meta['Macros'][e]) for e in FolderRoots: ContentHTML = ReplWithEsc(ContentHTML, f"[staticoso:Folder:{e}:AbsoluteRoot]", FolderRoots[e]) for e in Categories: ContentHTML = ReplWithEsc(ContentHTML, f"[staticoso:Category:{e}]", Categories[e]) + ContentHTML = ReplWithEsc(ContentHTML, f"[staticoso:Category:{e}]", Categories[e]) SlimHTML = HTMLPagesList + ContentHTML return HTML, ContentHTML, SlimHTML, Description, Image -def DoMinifyHTML(HTML): - return htmlmin.minify( - input=HTML, - remove_comments=True, - remove_empty_space=True, - remove_all_empty_space=False, - reduce_empty_attributes=True, - reduce_boolean_attributes=True, - remove_optional_attribute_quotes=True, - convert_charrefs=True, - keep_pre=True) - -def MakeSite(OutputDir, LimitFiles, TemplatesText, StaticPartsText, DynamicParts, DynamicPartsText, ConfMenu, GlobalMacros, SiteName, BlogName, SiteTagline, SiteTemplate, SiteDomain, SiteRoot, FolderRoots, SiteLang, Locale, Minify, NoScripts, ImgAltToTitle, ImgTitleToAlt, Sorting, MarkdownExts, AutoCategories): +def MakeSite(OutputDir, LimitFiles, TemplatesText, StaticPartsText, DynamicParts, DynamicPartsText, ConfMenu, GlobalMacros, SiteName, BlogName, SiteTagline, SiteTemplate, SiteDomain, SiteRoot, FolderRoots, SiteLang, Locale, Minify, MinifyKeepComments, NoScripts, ImgAltToTitle, ImgTitleToAlt, Sorting, MarkdownExts, AutoCategories): PagesPaths, PostsPaths, Pages, MadePages, Categories = [], [], [], [], {} for Ext in FileExtensions['Pages']: for File in Path('Pages').rglob(f"*.{Ext}"): @@ -435,15 +439,7 @@ def MakeSite(OutputDir, LimitFiles, TemplatesText, StaticPartsText, DynamicParts if not Exists: File = f"Categories/{Cat}.md" FilePath = f"{OutputDir}/{File}" - WriteFile(FilePath, f"""\ -// Title: {Cat} -// Type: Page -// Index: True - -# {Cat} - -
[staticoso:Category:{Cat}]
-""") + WriteFile(FilePath, CategoryPageTemplate.format(Title=Cat)) Content, Titles, Meta = PagePreprocessor(FilePath, SiteRoot) Pages += [[File, Content, Titles, Meta]] @@ -497,7 +493,7 @@ def MakeSite(OutputDir, LimitFiles, TemplatesText, StaticPartsText, DynamicParts Locale=Locale) if Minify: - HTML = DoMinifyHTML(HTML) + HTML = DoMinifyHTML(HTML, MinifyKeepComments) if NoScripts: HTML = StripTags(HTML, ['script']) if ImgAltToTitle or ImgTitleToAlt: diff --git a/Source/Modules/Utils.py b/Source/Modules/Utils.py index 10b2530..8c067f3 100644 --- a/Source/Modules/Utils.py +++ b/Source/Modules/Utils.py @@ -84,6 +84,7 @@ def FindAllIndex(Str, Sub): yield i i = Str.find(Sub, i+1) +# Replace substrings in a string, except when an escape char is prepended def ReplWithEsc(Str, Find, Repl, Esc='\\'): New = '' Sects = Str.split(Find) @@ -101,6 +102,11 @@ def ReplWithEsc(Str, Find, Repl, Esc='\\'): New += Repl + e return New +def DictReplWithEsc(Str, Dict, Esc='\\'): + for Item in Dict: + Str = ReplWithEsc(Str, Item, Dict[Item], Esc='\\') + return Str + def NumsFromFileName(Path): Name = Path.split('/')[-1] Split = len(Name)