From 58d9b58fd8d50548d807fc566560d2d52aed2de2 Mon Sep 17 00:00:00 2001
From: octospacc <octo@tutamail.com>
Date: Sat, 3 Sep 2022 17:48:39 +0200
Subject: [PATCH] Choos. log level + threads num.; Add HTML <pre> fix; Add md.
 extension

---
 Source/Build.py            | 49 +++++++++++++++++++++-----------------
 Source/Modules/HTML.py     | 12 ++++++++++
 Source/Modules/Logging.py  | 35 +++++++++++++++++++++++++++
 Source/Modules/Markdown.py |  2 +-
 Source/Modules/Site.py     | 38 +++++++++++++++++++++--------
 TODO                       |  9 ++++---
 6 files changed, 109 insertions(+), 36 deletions(-)
 create mode 100644 Source/Modules/Logging.py

diff --git a/Source/Build.py b/Source/Build.py
index 88340b9..6e9d926 100755
--- a/Source/Build.py
+++ b/Source/Build.py
@@ -20,11 +20,12 @@ try:
 	from Modules.ActivityPub import *
 	ActivityPub = True
 except:
-	print("[W] ⚠ Can't load the ActivityPub module. Its use is disabled. Make sure the 'requests' library is installed.")
+	logging.warning("⚠ Can't load the ActivityPub module. Its use is disabled. Make sure the 'requests' library is installed.")
 	ActivityPub = False
 
 from Modules.Config import *
 from Modules.Gemini import *
+from Modules.Logging import *
 from Modules.Markdown import *
 from Modules.Site import *
 from Modules.Sitemap import *
@@ -74,10 +75,10 @@ def CheckSafeOutDir(OutDir):
 	OutDir = os.path.realpath(OutDir)
 	OutFolder = OutDir.split('/')[-1]
 	if InDir == OutDir:
-		print(f"[E] ⛔ Output and Input directories ({OutDir}) can't be the same. Exiting.")
+		logging.error(f"⛔ Output and Input directories ({OutDir}) can't be the same. Exiting.")
 		exit(1)
 	elif OutFolder in ReservedPaths and f"{InDir}/{OutFolder}" == OutDir:
-		print(f"[E] ⛔ Output directory {OutDir} can't be a reserved subdirectory of the Input. Exiting.")
+		logging.error(f"⛔ Output directory {OutDir} can't be a reserved subdirectory of the Input. Exiting.")
 		exit(1)
 
 def GetModifiedFiles(OutDir):
@@ -113,16 +114,15 @@ def Main(Args, FeedEntries):
 
 	SiteName = Flags['SiteName'] = OptionChoose('', Args.SiteName, ReadConf(SiteConf, 'Site', 'Name'))
 	if SiteName:
-		print(f"[I] Compiling: {SiteName}")
+		logging.info(f"Compiling: {SiteName}")
 
 	OutDir = Flags['OutDir'] = OptionChoose('public', Args.OutputDir, ReadConf(SiteConf, 'Site', 'OutputDir'))
 	OutDir = Flags['OutDir'] = OutDir.removesuffix('/')
 	CheckSafeOutDir(OutDir)
-	print(f"[I] Outputting to: {OutDir}/")
+	logging.info(f"Outputting to: {OutDir}/")
 
-	Logging = Args.Logging
-	Threads = Args.Threads
-	DiffBuild = Args.DiffBuild
+	Threads = Args.Threads if Args.Threads else 0
+	DiffBuild = Args.DiffBuild if Args.DiffBuild else False
 
 	BlogName = Flags['BlogName'] = OptionChoose('', Args.BlogName, ReadConf(SiteConf, 'Site', 'BlogName'))
 	SiteTagline = Flags['SiteTagline'] = OptionChoose('', Args.SiteTagline, ReadConf(SiteConf, 'Site', 'Tagline'))
@@ -151,6 +151,7 @@ def Main(Args, FeedEntries):
 
 	ImgAltToTitle = Flags['ImgAltToTitle'] = StringBoolChoose(True, Args.ImgAltToTitle, ReadConf(SiteConf, 'Site', 'ImgAltToTitle'))
 	ImgTitleToAlt = Flags['ImgTitleToAlt'] = StringBoolChoose(False, Args.ImgTitleToAlt, ReadConf(SiteConf, 'Site', 'ImgTitleToAlt'))
+	HTMLFixPre = Flags['HTMLFixPre'] = StringBoolChoose(False, Args.HTMLFixPre, ReadConf(SiteConf, 'Site', 'HTMLFixPre'))
 
 	CategoriesAutomatic = Flags['CategoriesAutomatic'] = StringBoolChoose(False, Args.CategoriesAutomatic, ReadConf(SiteConf, 'Categories', 'Automatic'))
 	CategoriesUncategorized = Flags['CategoriesUncategorized'] = OptionChoose('Uncategorized', Args.CategoriesUncategorized, ReadConf(SiteConf, 'Categories', 'Uncategorized'))
@@ -176,10 +177,10 @@ def Main(Args, FeedEntries):
 	Locale = LoadLocale(SiteLang)
 
 	if DiffBuild:
-		print("[I] Build mode: Differential")
+		logging.info("Build mode: Differential")
 		LimitFiles = GetModifiedFiles(OutDir)
 	else:
-		print("[I] Build mode: Clean")
+		logging.info("Build mode: Clean")
 		ResetOutDir(OutDir)
 		LimitFiles = False
 
@@ -195,25 +196,26 @@ def Main(Args, FeedEntries):
 			shutil.copytree('Posts', f"{OutDir}.gmi/Posts", ignore=IgnoreFiles, dirs_exist_ok=True)
 
 	if not (HavePages or HavePosts):
-		print("[E] No Pages or posts found. Nothing to do, exiting!")
+		logging.error("⛔ No Pages or posts found. Nothing to do, exiting!")
 		exit(1)
 
-	print("[I] Generating HTML")
+	logging.info("Generating HTML")
 	Pages = MakeSite(
 		Flags=Flags,
 		LimitFiles=LimitFiles,
 		Snippets=Snippets,
 		ConfMenu=ConfMenu,
 		GlobalMacros=ReadConf(SiteConf, 'Macros'),
-		Locale=Locale)
+		Locale=Locale,
+		Threads=Threads)
 
 	if FeedEntries != 0:
-		print("[I] Generating Feeds")
+		logging.info("Generating Feeds")
 		for FeedType in (True, False):
 			MakeFeed(Flags, Pages, FeedType)
 
 	if ActivityPub and MastodonURL and MastodonToken and SiteDomain:
-		print("[I] Mastodon Stuff")
+		logging.info("Mastodon Stuff")
 		MastodonPosts = MastodonShare(Flags, Pages, Locale)
 	else:
 		MastodonPosts = []
@@ -235,24 +237,24 @@ def Main(Args, FeedEntries):
 		WriteFile(File, Content)
 
 	if Flags['GemtextOutput']:
-		print("[I] Generating Gemtext")
+		logging.info("Generating Gemtext")
 		GemtextCompileList(Flags, Pages, LimitFiles)
 
-	print("[I] Cleaning Temporary Files")
+	logging.info("Cleaning Temporary Files")
 	DelTmp(OutDir)
 
 	if Flags['SitemapOutput']:
-		print("[I] Generating Sitemap")
+		logging.info("Generating Sitemap")
 		MakeSitemap(Flags, Pages)
 
-	print("[I] Copying Assets")
+	logging.info("Copying Assets")
 	os.system(f"cp -R Assets/* {OutDir}/")
 
 if __name__ == '__main__':
 	StartTime = time.time()
 
 	Parser = argparse.ArgumentParser()
-	Parser.add_argument('--Logging', type=str) # Levels: Debug, Verbose, Info, Warning, Error.
+	Parser.add_argument('--Logging', type=str) # Levels: Debug, Info, Warning, Error.
 	Parser.add_argument('--Threads', type=str)
 	Parser.add_argument('--DiffBuild', type=str)
 	Parser.add_argument('--OutputDir', type=str)
@@ -269,6 +271,7 @@ if __name__ == '__main__':
 	Parser.add_argument('--NoScripts', type=str)
 	Parser.add_argument('--ImgAltToTitle', type=str)
 	Parser.add_argument('--ImgTitleToAlt', type=str)
+	Parser.add_argument('--HTMLFixPre', type=str)
 	Parser.add_argument('--GemtextOutput', type=str)
 	Parser.add_argument('--GemtextHeader', type=str)
 	Parser.add_argument('--SiteTagline', type=str)
@@ -286,15 +289,17 @@ if __name__ == '__main__':
 	Parser.add_argument('--CategoriesUncategorized', type=str)
 	Args = Parser.parse_args()
 
+	ConfigLogging(Args.Logging)
+
 	try:
 		import lxml
 		from Modules.Feed import *
 		FeedEntries = Args.FeedEntries if Args.FeedEntries else 'Default'
 	except:
-		print("[W] ⚠ Can't load the XML libraries. XML Feeds Generation is Disabled. Make sure the 'lxml' library is installed.")
+		logging.warning("⚠ Can't load the XML libraries. XML Feeds Generation is Disabled. Make sure the 'lxml' library is installed.")
 		FeedEntries = 0
 
 	Main(
 		Args=Args,
 		FeedEntries=FeedEntries)
-	print(f"[I] ✅ Done! ({round(time.time()-StartTime,3)}s)")
+	logging.info(f"✅ Done! ({round(time.time()-StartTime,3)}s)")
diff --git a/Source/Modules/HTML.py b/Source/Modules/HTML.py
index 1589b0b..34812d6 100644
--- a/Source/Modules/HTML.py
+++ b/Source/Modules/HTML.py
@@ -35,6 +35,18 @@ def StripTags(HTML, ToStrip): # Remove desired tags from the HTML
 			t.replace_with('')
 	return str(Soup)
 
+def DoHTMLFixPre(HTML):
+	if not ("<pre>" in HTML or "<pre " in HTML):
+		return HTML
+	Soup = MkSoup(HTML)
+	Tags = Soup.find_all('pre')
+	for t in Tags:
+		FirstLine = str(t).splitlines()[0].lstrip().rstrip()
+		if FirstLine.endswith('>'):
+			New = MkSoup(str(t).replace('\n', '', 1))
+			t.replace_with(New.pre)
+	return str(Soup)
+
 def WriteImgAltAndTitle(HTML, AltToTitle, TitleToAlt): # Adds alt or title attr. to <img> which only have one of them
 	Soup = MkSoup(HTML)
 	Tags = Soup.find_all('img')
diff --git a/Source/Modules/Logging.py b/Source/Modules/Logging.py
new file mode 100644
index 0000000..61f362e
--- /dev/null
+++ b/Source/Modules/Logging.py
@@ -0,0 +1,35 @@
+""" ================================= |
+| This file is part of                |
+|   staticoso                         |
+| Just a simple Static Site Generator |
+|                                     |
+| Licensed under the AGPLv3 license   |
+|   Copyright (C) 2022, OctoSpacc     |
+| ================================= """
+
+import logging
+import sys
+
+LoggingFormat = '[%(levelname)s] %(message)s'
+LoggingLevels = {
+	"Debug": {"Name":"D", "Num":15},
+	"Info": {"Name":"I", "Num":20},
+	"Warning": {"Name":"W", "Num":30},
+	"Error": {"Name":"E", "Num":40}}
+
+def SetupLogging(Level):
+	logging.basicConfig(format=LoggingFormat, stream=sys.stdout, level=Level)
+	logging.addLevelName(15, 'D') # Standard (10) Debug level makes third-party modules spam
+	logging.addLevelName(20, 'I')
+	logging.addLevelName(30, 'W')
+	logging.addLevelName(40, 'E')
+
+def ConfigLogging(Level):
+	Num = 20
+	if Level:
+		if Level.isdecimal():
+			Num = int(Level)
+		else:
+			if Level.lower() in LoggingLevels:
+				Num = LoggingLevels['Level']['Num']
+	SetupLogging(Num)
diff --git a/Source/Modules/Markdown.py b/Source/Modules/Markdown.py
index ec44c78..7e5a7cd 100644
--- a/Source/Modules/Markdown.py
+++ b/Source/Modules/Markdown.py
@@ -9,7 +9,7 @@
 
 from Libs.markdown import markdown
 
-MarkdownExtsDefault = ('attr_list', 'def_list', 'footnotes', 'md_in_html', 'tables')
+MarkdownExtsDefault = ('attr_list', 'def_list', 'fenced_code', 'footnotes', 'md_in_html', 'tables')
 
 def MarkdownHTMLEscape(Str, Extensions=()): # WIP
 	Text = ''
diff --git a/Source/Modules/Site.py b/Source/Modules/Site.py
index 418ca40..7316990 100644
--- a/Source/Modules/Site.py
+++ b/Source/Modules/Site.py
@@ -13,6 +13,7 @@ from Libs.bs4 import BeautifulSoup
 from Modules.Config import *
 from Modules.Elements import *
 from Modules.HTML import *
+from Modules.Logging import *
 from Modules.Markdown import *
 from Modules.Pug import *
 from Modules.Utils import *
@@ -128,10 +129,12 @@ def PagePreprocessor(Path, TempPath, Type, SiteTemplate, SiteRoot, GlobalMacros,
 				if ll.startswith('#') or (ll.startswith('<') and ll[1:].startswith(Headings)):
 					if ll.startswith('#'):
 						Title = ll
-						#Index = Title.split(' ')[0].count('#')
 					elif ll.startswith('<'):
-						#Index = int(ll[2])
-						Title = '#'*h + str(ll[3:])
+						if ll[3:].startswith((" class='NoTitle", ' class="NoTitle')):
+							Content += l + '\n'
+							continue
+						else:
+							Title = '#'*h + str(ll[3:])
 					DashTitle = DashifyTitle(MkSoup(Title.lstrip('#')).get_text(), DashyTitles)
 					DashyTitles += [DashTitle]
 					Titles += [Title]
@@ -213,7 +216,7 @@ def CanIndex(Index, For):
 	else:
 		return True if Index == For else False
 
-def PatchHTML(File, HTML, StaticPartsText, DynamicParts, DynamicPartsText, HTMLPagesList, PagePath, Content, Titles, Meta, SiteRoot, SiteName, BlogName, FolderRoots, Categories, SiteLang, Locale, LightRun):
+def PatchHTML(File, HTML, StaticPartsText, DynamicParts, DynamicPartsText, HTMLPagesList, PagePath, Content, Titles, Meta, SiteDomain, SiteRoot, SiteName, BlogName, FolderRoots, Categories, SiteLang, Locale, LightRun):
 	HTMLTitles = FormatTitles(Titles)
 	BodyDescription, BodyImage = '', ''
 	if not File.lower().endswith('.txt'):
@@ -228,6 +231,15 @@ def PatchHTML(File, HTML, StaticPartsText, DynamicParts, DynamicPartsText, HTMLP
 		if '<a class="footnote-ref"' in Content:
 			Content = AddToTagStartEnd(Content, '<a class="footnote-ref"', '</a>', '[', ']')
 
+		if ("<!-- noprocess />" or "<!--noprocess/>") in Content and ("</ noprocess -->" or "</noprocess-->") in Content:
+			Content = DictReplWithEsc(
+				Content, {
+					"<!-- noprocess />": "",
+					"<!--noprocess/>": "",
+					"</ noprocess -->": "",
+					"</noprocess-->": ""
+				})
+
 	Title = GetTitle(File.split('/')[-1], Meta, Titles, 'MetaTitle', BlogName)
 	Description = GetDescription(Meta, BodyDescription, 'MetaDescription')
 	Image = GetImage(Meta, BodyImage, 'MetaImage')
@@ -281,6 +293,7 @@ def PatchHTML(File, HTML, StaticPartsText, DynamicParts, DynamicPartsText, HTMLP
 				'<staticoso:PageContentInfo>': MakeContentHeader(Meta, Locale, MakeCategoryLine(File, Meta)),
 				'[staticoso:BuildTime]': datetime.now().strftime('%Y-%m-%d %H:%M'),
 				'<staticoso:BuildTime>': datetime.now().strftime('%Y-%m-%d %H:%M'),
+				'<staticoso:SiteDomain>': SiteDomain,
 				'[staticoso:Site:Name]': SiteName,
 				'<staticoso:SiteName>': SiteName,
 				'[staticoso:Site:AbsoluteRoot]': SiteRoot,
@@ -370,6 +383,7 @@ def HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, L
 		Content=Content,
 		Titles=Titles,
 		Meta=Meta,
+		SiteDomain=SiteDomain,
 		SiteRoot=SiteRoot,
 		SiteName=SiteName,
 		BlogName=BlogName,
@@ -383,7 +397,7 @@ def HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, L
 		if not LightRun:
 			HTML = DoMinifyHTML(HTML, MinifyKeepComments)
 		ContentHTML = DoMinifyHTML(ContentHTML, MinifyKeepComments)
-	if Flags['NoScripts']:
+	if Flags['NoScripts'] and ("<script" in ContentHTML or "<script" in HTML):
 		if not LightRun:
 			HTML = StripTags(HTML, ['script'])
 		ContentHTML = StripTags(ContentHTML, ['script'])
@@ -391,6 +405,10 @@ def HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, L
 		if not LightRun:
 			HTML = WriteImgAltAndTitle(HTML, ImgAltToTitle, ImgTitleToAlt)
 		ContentHTML = WriteImgAltAndTitle(ContentHTML, ImgAltToTitle, ImgTitleToAlt)
+	if Flags['HTMLFixPre']:
+		if not LightRun:
+			HTML = DoHTMLFixPre(HTML)
+		ContentHTML = DoHTMLFixPre(ContentHTML)
 
 	if LightRun:
 		SlimHTML = None
@@ -408,9 +426,9 @@ def MultiprocHandlePage(d):
 def MultiprocPagePreprocessor(d):
 	return PagePreprocessor(d['Path'], d['TempPath'], d['Type'], d['Template'], d['SiteRoot'], d['GlobalMacros'], d['CategoryUncategorized'], d['LightRun'])
 
-def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale):
+def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale, Threads):
 	PagesPaths, PostsPaths, Pages, MadePages, Categories = [], [], [], [], {}
-	PoolSize = cpu_count()
+	PoolSize = cpu_count() if Threads <= 0 else Threads
 	OutDir, MarkdownExts, Sorting, MinifyKeepComments = Flags['OutDir'], Flags['MarkdownExts'], Flags['Sorting'], Flags['MinifyKeepComments']
 	SiteName, BlogName, SiteTagline = Flags['SiteName'], Flags['BlogName'], Flags['SiteTagline']
 	SiteTemplate, SiteLang = Flags['SiteTemplate'], Flags['SiteLang']
@@ -432,7 +450,7 @@ def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale):
 	if Sorting['Posts'] == 'Inverse':
 		PostsPaths.reverse()
 
-	print("[I] Preprocessing Source Pages")
+	logging.info("Preprocessing Source Pages")
 	MultiprocPages = []
 	for Type in ['Page', 'Post']:
 		if Type == 'Page':
@@ -454,7 +472,7 @@ def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale):
 	PugCompileList(OutDir, Pages, LimitFiles)
 
 	if Categories:
-		print("[I] Generating Category Lists")
+		logging.info("Generating Category Lists")
 		for Cat in Categories:
 			for Type in ('Page', 'Post'):
 				Categories[Cat] += GetHTMLPagesList(
@@ -488,7 +506,7 @@ def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale):
 			if e == File:
 				ConfMenu[i] = None
 
-	print("[I] Writing Pages")
+	logging.info("Writing Pages")
 	MultiprocPages = []
 	for Page in Pages:
 		MultiprocPages += [{'Flags':Flags, 'Page':Page, 'Pages':Pages, 'Categories':Categories, 'LimitFiles':LimitFiles, 'Snippets':Snippets, 'ConfMenu':ConfMenu, 'Locale':Locale}]
diff --git a/TODO b/TODO
index 9b0b283..11ae50b 100644
--- a/TODO
+++ b/TODO
@@ -1,6 +1,9 @@
+- Custom path for Posts
+- Support for YAML header in Markdown
+- Support for HTML comment lines (<!-- -->) in any format
+- Support for rST and AsciiDoc (?)
 - Showing build percentage as dots
-- Custom thread number, 0 to disable multithreading
-- Posts in draft state (will not be compiled)
+- Posts in draft state (will not be compiled) / show unlisted status for posts with Index = False
 - Check if external tools (pug-cli, html2gmi) are installed
 - Static code syntax highlighing
 - Override internal HTML snippets (meta lines, page lists, ...) with config file in Templates/NAME.ini
@@ -8,7 +11,7 @@
 - Show page size/words/time in meta line
 - Add feed support for diary-like pages
 - Fix excess whitespace in some section/menu titles
-- Change staticoso service tag enclosure from [] to <>
+- Change all staticoso service tag enclosures from [] to <>
 - Investigate a strange bug with Macros
 - Handle file extensions with any case sensitivity, not just lowercase; currently the bulk of the issue is finding the files on disk
 - Test sorting by date for files not starting with date, and dated folders