Add Sites/Main; Upd. build scripts; Refactoring (Site.py); Some work on transclusions

2023-02-27 18:11:16 +01:00 · 2023-02-27 18:11:16 +01:00 · 0567297de6
parent a342f88e3c
commit 0567297de6
10 changed files with 210 additions and 100 deletions
--- a/App/Source/Modules/Site.py
+++ b/App/Source/Modules/Site.py
@ -19,10 +19,62 @@ from Modules.Meta import *
 from Modules.Pug import *
 from Modules.Utils import *

-def PatchHTML(Flags:dict, File, HTML:str, Snippets:dict, HTMLPagesList:str, PagePath:str, Content:str, Titles:list, Meta:dict, Categories, Locale:dict, LightRun):
-	f = NameSpace(Flags)
+def HandleStaticParts(Html:str, Snippets:dict):
+	for e in Snippets['StaticParts']:
+		Html = ReplWithEsc(Html, f"[staticoso:StaticPart:{e}]", Snippets['StaticParts'][e])
+		Html = ReplWithEsc(Html, f"<staticoso:StaticPart:{e}>", Snippets['StaticParts'][e])
+	return Html

-	HTMLTitles = FormatTitles(Titles)
+def HandleDynamicParts(Flags:dict, Html:str, Snippets:dict):
+	f = NameSpace(Flags)
+	Key = 'staticoso:dynamicpart'
+	if f'{Key}:' in Html.lower(): # Reduce unnecessary cycles
+		for Line in Html.splitlines():
+			Line = Line.lstrip().rstrip()
+			LineLow = Line.lower()
+			if (LineLow.startswith(f'[{Key}:') and LineLow.endswith(']')) or (LineLow.startswith(f'<{Key}:') and LineLow.endswith('>')):
+				Path =  Line[len(f'<{Key}:'):-1]
+				Section = Path.split('/')[-1]
+				if Section in f.DynamicParts:
+					Parts = f.DynamicParts[Section]
+					Text = ''
+					Parts = SureList(Parts)
+					for Part in Parts:
+						Text += Snippets['DynamicParts'][f'{Path}/{Part}'] + '\n'
+				else:
+					Text = ''
+				Html = ReplWithEsc(Html, f'[staticoso:DynamicPart:{Path}]', Text)
+				Html = ReplWithEsc(Html, f'<staticoso:DynamicPart:{Path}>', Text)
+	return Html
+
+# TODO: This would need to be handled either fully before or fully after after all pages' content has been transformed to HTML, else other markups end up in HTML and the page is broken
+def HandleTransclusions(Html:str, Caller:str, Pages:list):
+	#if Type == 'Evals': # [% cmd %] | {% cmd %}
+	Targets = []
+	Finding = Html
+	Start = Finding.find('{{')
+	while Start != -1:
+		Start = Start + 2
+		Finding = Finding[Start:]
+		Stop = Finding.find('}}')
+		if Stop != -1:
+			Targets += [Finding[:Stop]]
+		Start = Finding.find('{{')
+	for Target in Targets:
+		# Maybe we should show an error message on possible recursive transclusion, as currently this doesn't handle escaped tokens
+		if Target != Caller:
+			for File, Content, _, _ in Pages:
+				if File == Target:
+					Html = ReplWithEsc(Html, '{{' + Target + '}}', Content)
+					break
+	return Html
+
+def PatchHtml(Flags:dict, Pages:list, Page:dict, Context:dict, Snippets:dict, Locale:dict, LightRun):
+	f = NameSpace(Flags)
+	File, PagePath, Content, Titles, Meta = tuple(Page.values())
+	Html, HtmlPagesList, Categories = tuple(Context.values())
+
+	HtmlTitles = FormatTitles(Titles)
 	BodyDescription, BodyImage = '', ''
 	if not File.lower().endswith('.txt'):
 		Soup = MkSoup(Content)
@ -59,50 +111,29 @@ def PatchHTML(Flags:dict, File, HTML:str, Snippets:dict, HTMLPagesList:str, Page
 	TimeNow = datetime.now().strftime('%Y-%m-%d %H:%M')
 	RelativeRoot = GetPathLevels(PagePath)

-	if 'staticoso:DynamicPart:' in HTML: # Reduce risk of unnecessary cycles
-		for Line in HTML.splitlines():
-			Line = Line.lstrip().rstrip()
-			if (Line.startswith('[staticoso:DynamicPart:') and Line.endswith(']')) or (Line.startswith('<staticoso:DynamicPart:') and Line.endswith('>')):
-				Path =  Line[len('<staticoso:DynamicPart:'):-1]
-				Section = Path.split('/')[-1]
-				if Section in f.DynamicParts:
-					Part = f.DynamicParts[Section]
-					Text = ''
-					if type(Part) == list:
-						for e in Part:
-							Text += Snippets['DynamicParts'][f"{Path}/{e}"] + '\n'
-					elif type(Part) == str:
-						Text = Snippets['DynamicParts'][f"{Path}/{Part}"]
-				else:
-					Text = ''
-				HTML = ReplWithEsc(HTML, f"[staticoso:DynamicPart:{Path}]", Text)
-				HTML = ReplWithEsc(HTML, f"<staticoso:DynamicPart:{Path}>", Text)
-
-	for i in range(2):
-		for e in Snippets['StaticParts']:
-			HTML = ReplWithEsc(HTML, f"[staticoso:StaticPart:{e}]", Snippets['StaticParts'][e])
-			HTML = ReplWithEsc(HTML, f"<staticoso:StaticPart:{e}>", Snippets['StaticParts'][e])
+	Html = WhileFuncResultChanges(HandleDynamicParts, {"Flags": Flags, "Html": Html, "Snippets": Snippets}, 'Html')
+	Html = WhileFuncResultChanges(HandleStaticParts, {"Html": Html, "Snippets": Snippets}, 'Html')

 	if LightRun:
-		HTML = None
+		Html = None
 	else:
-		HTML = WrapDictReplWithEsc(HTML, {
+		Html = WrapDictReplWithEsc(Html, {
 			#'[staticoso:PageHead]': Meta['Head'],
 			#'<staticoso:PageHead>': Meta['Head'],
 			# #DEPRECATION #
-			'staticoso:Site:Menu': HTMLPagesList,
+			'staticoso:Site:Menu': HtmlPagesList,
 			'staticoso:Page:Lang': Meta['Language'] if Meta['Language'] else f.SiteLang,
-			'staticoso:Page:Chapters': HTMLTitles,
+			'staticoso:Page:Chapters': HtmlTitles,
 			'staticoso:Page:Title': Title,
 			'staticoso:Page:Description': Description,
 			'staticoso:Page:Image': Image,
 			'staticoso:Page:Path': PagePath,
 			'staticoso:Page:Style': Meta['Style'],
 			################
-			'staticoso:SiteMenu': HTMLPagesList,
+			'staticoso:SiteMenu': HtmlPagesList,
 			'staticoso:PageLang': Meta['Language'] if Meta['Language'] else f.SiteLang,
 			'staticoso:PageLanguage': Meta['Language'] if Meta['Language'] else f.SiteLang,
-			'staticoso:PageSections': HTMLTitles,
+			'staticoso:PageSections': HtmlTitles,
 			'staticoso:PageTitle': Title,
 			'staticoso:PageDescription': Description,
 			'staticoso:PageImage': Image,
@ -126,23 +157,24 @@ def PatchHTML(Flags:dict, File, HTML:str, Snippets:dict, HTMLPagesList:str, Page
 			'staticoso:SiteAbsoluteRoot': f.SiteRoot,
 			'staticoso:SiteRelativeRoot': RelativeRoot,
 		}, InternalMacrosWraps)
+		#Html = WhileFuncResultChanges(HandleTransclusions, {"Html": Html, "Caller": File, "Pages": Pages}, 'Html')
 		for e in Meta['Macros']:
-			HTML = ReplWithEsc(HTML, f"[:{e}:]", Meta['Macros'][e])
+			Html = ReplWithEsc(Html, f"[:{e}:]", Meta['Macros'][e])
 		for e in f.FolderRoots:
-			HTML = WrapDictReplWithEsc(HTML, {
+			Html = WrapDictReplWithEsc(Html, {
 				f'staticoso:CustomPath:{e}': f.FolderRoots[e],
 				f'staticoso:Folder:{e}:AbsoluteRoot': f.FolderRoots[e], #DEPRECATED
 			}, InternalMacrosWraps)
 		for e in Categories:
-			HTML = WrapDictReplWithEsc(HTML, {
+			Html = WrapDictReplWithEsc(Html, {
 				f'staticoso:Category:{e}': Categories[e],
 				f'staticoso:CategoryList:{e}': Categories[e],
 			}, InternalMacrosWraps)
-			HTML = ReplWithEsc(HTML, f'<span>[staticoso:Category:{e}]</span>', Categories[e]) #DEPRECATED
+			Html = ReplWithEsc(Html, f'<span>[staticoso:Category:{e}]</span>', Categories[e]) #DEPRECATED

 	# TODO: Clean this doubling?
-	ContentHTML = Content
-	ContentHTML = WrapDictReplWithEsc(ContentHTML, {
+	ContentHtml = Content
+	ContentHtml = WrapDictReplWithEsc(ContentHtml, {
 		# #DEPRECATION #
 		'[staticoso:Page:Title]': Title,
 		'[staticoso:Page:Description]': Description,
@ -157,21 +189,22 @@ def PatchHTML(Flags:dict, File, HTML:str, Snippets:dict, HTMLPagesList:str, Page
 		'<staticoso:SiteAbsoluteRoot>': f.SiteRoot,
 		'<staticoso:SiteRelativeRoot>': RelativeRoot,
 	}, InternalMacrosWraps)
+	#Html = WhileFuncResultChanges(HandleTransclusions, {"Html": Html, "Caller": File, "Pages": Pages}, 'Html')
 	for e in Meta['Macros']:
-		ContentHTML = ReplWithEsc(ContentHTML, f"[:{e}:]", Meta['Macros'][e])
+		ContentHtml = ReplWithEsc(ContentHtml, f"[:{e}:]", Meta['Macros'][e])
 	for e in f.FolderRoots:
-		ContentHTML = WrapDictReplWithEsc(ContentHTML, {
+		ContentHtml = WrapDictReplWithEsc(ContentHtml, {
 			f'staticoso:CustomPath:{e}': f.FolderRoots[e],
 			f'staticoso:Folder:{e}:AbsoluteRoot': f.FolderRoots[e], #DEPRECATED
 		}, InternalMacrosWraps)
 	for e in Categories:
-		ContentHTML = WrapDictReplWithEsc(ContentHTML, {
+		ContentHtml = WrapDictReplWithEsc(ContentHtml, {
 			f'staticoso:Category:{e}': Categories[e],
 			f'staticoso:CategoryList:{e}': Categories[e],
 		}, InternalMacrosWraps)
-		ContentHTML = ReplWithEsc(ContentHTML, f'<span>[staticoso:Category:{e}]</span>', Categories[e]) #DEPRECATED
+		ContentHtml = ReplWithEsc(ContentHtml, f'<span>[staticoso:Category:{e}]</span>', Categories[e]) #DEPRECATED

-	return HTML, ContentHTML, Description, Image
+	return Html, ContentHtml, Description, Image

 def BuildPagesSearch(Flags:dict, Pages:list, Template:str, Snippets:dict, Locale:dict):
 	SearchContent = ''
@ -187,21 +220,16 @@ def BuildPagesSearch(Flags:dict, Pages:list, Template:str, Snippets:dict, Locale
 				{Page["ContentHtml"]}
 			</div>
 		'''
-	return PatchHTML(
+	return PatchHtml(
 		Flags=Flags,
-		File='Search.html',
-		HTML=Template,
+		Pages=[],
+		Page={"File": "Search.html", "PagePath": "Search.html", "Content": Base[0] + SearchContent + Base[1], "Titles": [], "Meta": PageMetaDefault},
+		Context={"Html": Template, "HtmlPagesList": "", "Categories": []},
 		Snippets=Snippets,
-		HTMLPagesList='',
-		PagePath='Search.html',
-		Content=Base[0] + SearchContent + Base[1],
-		Titles=[],
-		Meta=PageMetaDefault,
-		Categories=[],
 		Locale=Locale,
 		LightRun=False)[0]

-def HandlePage(Flags:dict, Page:list, Pages, Categories, LimitFiles, Snippets:dict, ConfMenu, Locale:dict):
+def HandlePage(Flags:dict, Page:list, Pages:list, Categories, LimitFiles, Snippets:dict, ConfMenu, Locale:dict):
 	File, Content, Titles, Meta = Page
 	f = NameSpace(Flags)
 	TemplatesText = Snippets['Templates']
@ -233,17 +261,12 @@ def HandlePage(Flags:dict, Page:list, Pages, Categories, LimitFiles, Snippets:di
 			For='Menu',
 			MenuStyle=TemplateMeta['MenuStyle'])

-	HTML, ContentHTML, Description, Image = PatchHTML(
+	HTML, ContentHTML, Description, Image = PatchHtml(
 		Flags,
-		File=File,
-		HTML=TemplatesText[Meta['Template']],
+		Pages=Pages,
+		Page={"File": File, "PagePath": PagePath[len(f"{f.OutDir}/"):], "Content": Content, "Titles": Titles, "Meta": Meta},
+		Context={"Html": TemplatesText[Meta['Template']], "HtmlPagesList": HTMLPagesList, "Categories": Categories},
 		Snippets=Snippets,
-		HTMLPagesList=HTMLPagesList,
-		PagePath=PagePath[len(f"{f.OutDir}/"):],
-		Content=Content,
-		Titles=Titles,
-		Meta=Meta,
-		Categories=Categories,
 		Locale=Locale,
 		LightRun=LightRun)

@ -298,17 +321,12 @@ def HandlePage(Flags:dict, Page:list, Pages, Categories, LimitFiles, Snippets:di
 		WriteFile(ContentPagePath, ContentHTML)

 	if not LightRun and 'htmljournal' in ContentHTML.lower(): # Avoid extra cycles
-		HTML, _, _, _ = PatchHTML(
+		HTML, _, _, _ = PatchHtml(
 			Flags,
-			File=File,
-			HTML=TemplatesText[Meta['Template']],
+			Pages=Pages,
+			Page={"File": File, "PagePath": f'{StripExt(File)}.Journal.html', "Content": MakeHTMLJournal(Flags, Locale, f'{StripExt(File)}.html', ContentHTML), "Titles": "", "Meta": Meta},
+			Context={"Html": TemplatesText[Meta['Template']], "HtmlPagesList": HTMLPagesList, "Categories": Categories},
 			Snippets=Snippets,
-			HTMLPagesList=HTMLPagesList,
-			PagePath=f'{StripExt(File)}.Journal.html',
-			Content=MakeHTMLJournal(Flags, Locale, f'{StripExt(File)}.html', ContentHTML),
-			Titles='',
-			Meta=Meta,
-			Categories=Categories,
 			Locale=Locale,
 			LightRun=LightRun)
 		if Flags["JournalRedirect"]:
@ -338,7 +356,7 @@ def ReorderPagesPaths(Paths:dict, Sorting:dict):
 			Paths[Type].reverse()
 	return Paths

-def PopulateCategoryLists(Flags:dict, Pages:list, Categories):
+def PopulateCategoryLists(Flags:dict, Pages:list, Categories:dict):
 	for Cat in Categories:
 		for Type in ('Page', 'Post'):
 			Categories[Cat] += GetHTMLPagesList(
@ -424,23 +442,4 @@ def MakeSite(Flags:dict, LimitFiles, Snippets, ConfMenu, GlobalMacros:dict, Loca
 	logging.info("Writing Pages")
 	MadePages = WriteProcessedPages(Flags, Pages, Categories, ConfMenu, Snippets, LimitFiles, PoolSize, Locale)

-	# Do page transclusions here (?)
-	#while True:
-	#	Operated = False
-	#	for di,Dest in enumerate(MadePages):
-	#		#print(Dest[0])
-	#		#TempPath = f'{PathPrefix}{Dest["File"]}'
-	#		#LightRun = False if LimitFiles == False or TempPath in LimitFiles else True
-	#		#if not LightRun:
-	#		if '[staticoso:Transclude:' in Dest[4] and (LimitFiles == False or f'{PathPrefix}{Dest[0]}' in LimitFiles):
-	#			for Item in MadePages:
-	#				SrcPrefix = '' if Item[0].startswith('Posts/') else 'Pages/'
-	#				print(SrcPrefix, Item[0])
-	#				if Item[0] != Dest[0] and f'[staticoso:Transclude:{SrcPrefix}{Item[0]}]' in Dest[4]:
-	#					MadePages[di][4] = ReplWithEsc(Dest[4], f'<staticoso:Transclude:{Item[0]}>', Item[4])
-	#					print(f'[staticoso:Transclude:{SrcPrefix}{Item[0]}]', Item[4])
-	#					Operated = True
-	#	if not Operated:
-	#		break
-
 	return MadePages
--- a/App/Source/Modules/Utils.py
+++ b/App/Source/Modules/Utils.py
@ -92,16 +92,24 @@ def FindAllIndex(Str:str, Sub:str):
 def ReplWithEsc(Str:str, Find:str, Repl:str, Esc:str='\\'):
 	New = ''
 	Sects = Str.split(Find)
+	# Every time a substring is found
 	for i,e in enumerate(Sects):
+		# If it's the first split, append it directly to the New string
 		if i == 0:
 			New += e
+		# Wrapping parts of the escaped substrings in HTML tags is done to avoid multiple calls of this function nullifying escaping
 		elif i > 0:
+			# If prev. split ends with 2 times the escape (= escaping of the escape)
 			if Sects[i-1].endswith(Esc*2):
-				New = New[:-1]
+				Wrap = f'<span>{New[-1]}</span>'
+				New = New[:-2] + Wrap
 				New += Repl + e
+			# If prev. split ends with 1 time the escape (escaping of the substring)
 			elif Sects[i-1].endswith(Esc):
 				New = New[:-1]
-				New += Find + e
+				Wrap = f'<span>{Find[0]}</span>'
+				New += Wrap + Find[1:] + e
+			# If no escape char
 			else:
 				New += Repl + e
 	return New
@ -180,20 +188,30 @@ def PrintProcPercentDots(Proc:dict, DivMult=1):
 		return True
 	return False

-def MultiProcFunctWrap(Args:dict):
+def MultiProcFuncWrap(Args:dict):
 	PrintProcPercentDots(Args['Process'])
-	return Args['Process']['Funct'](Args)
+	return Args['Process']['Func'](Args)

-def DoMultiProc(Funct, ArgsCollection:list, Threads:int=cpu_count(), Progress:bool=False):
+def DoMultiProc(Func, ArgsCollection:list, Threads:int=cpu_count(), Progress:bool=False):
 	FinalArgsCollection = []
 	for Index, Args in enumerate(ArgsCollection):
 		FinalArgsCollection.append(Args)
-		FinalArgsCollection[Index].update({"Process": {"Funct": Funct, "Num": Index, "Count": len(ArgsCollection)}})
+		FinalArgsCollection[Index].update({"Process": {"Func": Func, "Num": Index, "Count": len(ArgsCollection)}})
 	Results = []
 	if Progress:
 		os.system('printf "["') # Using system print because (see PrintProcPercentDots())
 	with Pool(Threads) as MultiprocPool:
-		Results = MultiprocPool.map(MultiProcFunctWrap if Progress else Funct, FinalArgsCollection)
+		Results = MultiprocPool.map(MultiProcFuncWrap if Progress else Func, FinalArgsCollection)
 	if Progress:
 		os.system('printf "]\n"') # Newline after percentage dots
 	return Results
+
+# Execute a function, whose output is compared to one input argument, as long as the output is different from the previous cycle; the moment it's equal, return
+def WhileFuncResultChanges(Func, Args:dict, ResultKey:str):
+	Result = Args[ResultKey]
+	while True:
+		ResultOld = Result
+		Args.update({ResultKey: Result})
+		Result = Func(**Args)
+		if ResultOld == Result:
+			return Result
--- a/README.md
+++ b/README.md
@ -106,3 +106,4 @@ All of this is because some crucial things might be changed from one commit to a
 - Despite differential building and multithreading, the program still needs some more optimizations.
 - Ordering pages in the global menu with external configuration flags (outside the pages' source) yields broken and unpredictable results.
 - If site is small, the graphic of percentage completion is bugged (appears shorter).
+- Markdown markup in headings (e.g. `## Title **text**`) is not processed and will render literally (e.g. `<h2>Title **text**</h2>`)
--- a/Sites/BuildAll.sh
+++ b/Sites/BuildAll.sh
@ -4,7 +4,8 @@ cd "$( dirname "$( realpath "$0" )" )"
 # First build all sites independently,
 # then move them to a base public folder

-mkdir ../public
+rm -rf ../public/*
+mkdir -p ../public

 for Site in *
 do
@ -12,7 +13,10 @@ do
 	then
 		cd $Site
 		python3 ../../App/Source/Build.py
-		mv ./public ../../public/$Site
+		cp -r ./public ../../public/$Site
 		cd ..
 	fi
 done
+
+cd ../public
+mv ./Main/* ./
--- a/Sites/Docs/LICENSE.md
+++ b/Sites/Docs/LICENSE.md
--- a/Sites/Main/.github/workflows/main.yml
+++ b/Sites/Main/.github/workflows/main.yml
@ -0,0 +1,33 @@
+name: Build and Deploy with staticoso
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+  workflow_dispatch:
+
+jobs:
+  page_build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Build
+      run: |
+        sudo apt update
+        sudo apt install -y software-properties-common
+        sudo add-apt-repository ppa:deadsnakes/ppa
+        sudo apt update
+        sudo apt install -y python3.10 curl git
+        curl -sS https://bootstrap.pypa.io/get-pip.py | sudo python3.10
+        sudo pip3 install lxml
+        git clone --depth 1 https://gitlab.com/octtspacc/staticoso
+        python3.10 ./staticoso/Source/Build.py
+
+    - name: Deploy
+      uses: JamesIves/github-pages-deploy-action@v4
+      with:
+        folder: public
--- a/Sites/Main/.gitlab-ci.yml
+++ b/Sites/Main/.gitlab-ci.yml
@ -0,0 +1,16 @@
+image: alpine:latest
+
+before_script: |
+  apk update
+  apk add python3 git
+
+pages:
+  stage: deploy
+  script: |
+    git clone --depth 1 https://gitlab.com/octtspacc/staticoso
+    python3 ./staticoso/Source/Build.py
+  artifacts:
+    paths:
+    - public
+  rules:
+    - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
--- a/Sites/Main/Pages/index.md
+++ b/Sites/Main/Pages/index.md
@ -0,0 +1,16 @@
+// % Title = staticoso
+
+## Welcome to the always-WIP world of staticoso...
+
+Things you can do now:
+
+<div markdown="1">
+Cry
+: [Read the Docs](./Docs) - you cry here because they're outdated and incomplete
+: [Look at the code](https://gitlab.com/octtspacc/staticoso) - you cry here because it's bad
+</div>
+
+<div markdown="1">
+Procrastinate
+: [Visit my _sitoctt_](https://sitoctt.octt.eu.org) - involuntarily the flagship staticoso site
+</div>
--- a/Sites/Main/Site.ini
+++ b/Sites/Main/Site.ini
@ -0,0 +1,2 @@
+[Site]
+Name = staticoso
--- a/Sites/Main/Templates/Default.html
+++ b/Sites/Main/Templates/Default.html
@ -0,0 +1,21 @@
+<!DOCTYPE HTML>
+<html lang="[staticoso:Page:Lang]">
+<head>
+	<meta charset="UTF-8">
+	<meta name="viewport" content="width=device-width, initial-scale=1.0">
+	<link rel="stylesheet" href="[staticoso:Site:RelativeRoot]Assets/staticoso-docs.css">
+	<title>[staticoso:Page:Title] - [staticoso:Site:Name]</title>
+	<meta name="description" content="[staticoso:Page:Description]">
+	<link href="[staticoso:Site:RelativeRoot]favicon.png" rel="icon" type="image/png">
+	<meta property="og:type" content="website">
+	<meta property="og:title" content="[staticoso:Page:Title] - [staticoso:Site:Name]">
+	<meta property="og:description" content="[staticoso:Page:Description]">
+	<meta property="og:image" content="[staticoso:Page:Image]">
+	<style>
+		[staticoso:Page:Style]
+	</style>
+</head>
+<body>
+	[staticoso:Page:Content]
+</body>
+</html>