Multithreading yay

README.md

@@ -40,7 +40,9 @@ Needed for Gemtext output support:
 
 ## Features roadmap
 
-- [ ] Multithreading
+- [ ] Showing percentage of build process
+- [x] Multithreading
+- [x] Differential building
 - [ ] Overriding internal HTML snippets for template-specific ones
 - [ ] Static syntax highlighing for code blocks in any page
 - [x] File name used as a title for pages without one
@@ -65,7 +67,6 @@ Needed for Gemtext output support:
 - [x] Custom and automatic page sorting
 - [ ] SCSS compilation support for CSS templates
 - [ ] Pug support for base templates and page side parts
-- [ ] Differential recompile (to optimize resource waste on non-ephemeral servers)
 - [ ] Hot-recompile (for website development)
 - [x] TXT sitemap generation
 - [x] Atom + RSS feed generation

Source/Build.py

@@ -120,6 +120,10 @@ def Main(Args, FeedEntries):
 	CheckSafeOutDir(OutDir)
 	print(f"[I] Outputting to: {OutDir}/")
 
+	Logging = Args.Logging
+	Threads = Args.Threads
+	DiffBuild = Args.DiffBuild
+
 	BlogName = Flags['BlogName'] = OptionChoose('', Args.BlogName, ReadConf(SiteConf, 'Site', 'BlogName'))
 	SiteTagline = Flags['SiteTagline'] = OptionChoose('', Args.SiteTagline, ReadConf(SiteConf, 'Site', 'Tagline'))
 	SiteTemplate = Flags['SiteTemplate'] = OptionChoose('Default.html', Args.SiteTemplate, ReadConf(SiteConf, 'Site', 'Template'))
@@ -132,7 +136,6 @@ def Main(Args, FeedEntries):
 
 	NoScripts = Flags['NoScripts'] = StringBoolChoose(False, Args.NoScripts, ReadConf(SiteConf, 'Site', 'NoScripts'))
 	FolderRoots = Flags['FolderRoots'] = literal_eval(Args.FolderRoots) if Args.FolderRoots else {}
-	DiffBuild = Args.DiffBuild
 
 	ActivityPubTypeFilter = Flags['ActivityPubTypeFilter'] = OptionChoose('Post', Args.ActivityPubTypeFilter, ReadConf(SiteConf, 'ActivityPub', 'TypeFilter'))
 	ActivityPubHoursLimit = Flags['ActivityPubHoursLimit'] = OptionChoose(168, Args.ActivityPubHoursLimit, ReadConf(SiteConf, 'ActivityPub', 'HoursLimit'))
@@ -250,6 +253,7 @@ if __name__ == '__main__':
 
 	Parser = argparse.ArgumentParser()
 	Parser.add_argument('--Logging', type=str) # Levels: Debug, Verbose, Info, Warning, Error.
+	Parser.add_argument('--Threads', type=str)
 	Parser.add_argument('--DiffBuild', type=str)
 	Parser.add_argument('--OutputDir', type=str)
 	#Parser.add_argument('--InputDir', type=str)
@@ -293,4 +297,4 @@ if __name__ == '__main__':
 	Main(
 		Args=Args,
 		FeedEntries=FeedEntries)
-	print(f"[I] ✅ Done! ({round(time.process_time(),3)} / {round(time.time()-StartTime,3)} s)")
+	print(f"[I] ✅ Done! ({round(time.time()-StartTime,3)}s)")

Source/Modules/Site.py

@@ -84,7 +84,7 @@ def TemplatePreprocessor(Text):
 			Meta.update({i:MetaDefault[i]})
 	return Meta
 
-def PagePreprocessor(Path, Type, SiteTemplate, SiteRoot, GlobalMacros, CategoryUncategorized, LightRun=False):
+def PagePreprocessor(Path, TempPath, Type, SiteTemplate, SiteRoot, GlobalMacros, CategoryUncategorized, LightRun=False):
 	File = ReadFile(Path)
 	Path = Path.lower()
 	Content, Titles, DashyTitles, HTMLTitlesFound, Macros, Meta, MetaDefault = '', [], [], False, '', '', {
@@ -179,7 +179,8 @@ def PagePreprocessor(Path, Type, SiteTemplate, SiteRoot, GlobalMacros, CategoryU
 	if GlobalMacros:
 		Meta['Macros'].update(GlobalMacros)
 	Meta['Macros'].update(ReadConf(LoadConfStr('[Macros]\n' + Macros), 'Macros'))
-	return Content, Titles, Meta
+	#PrintPercentDots(ProcPercent)
+	return [TempPath, Content, Titles, Meta]
 
 def PagePostprocessor(FileType, Text, Meta):
 	for e in Meta['Macros']:
@@ -336,7 +337,7 @@ def HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, L
 			For='Menu',
 			MarkdownExts=MarkdownExts,
 			MenuStyle=TemplateMeta['MenuStyle'])
-	
+
 	HTML, ContentHTML, Description, Image = PatchHTML(
 		File=File,
 		HTML=TemplatesText[Meta['Template']],
@@ -369,21 +370,26 @@ def HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, L
 		if not LightRun:
 			HTML = WriteImgAltAndTitle(HTML, ImgAltToTitle, ImgTitleToAlt)
 		ContentHTML = WriteImgAltAndTitle(ContentHTML, ImgAltToTitle, ImgTitleToAlt)
-	
+
 	if LightRun:
 		SlimHTML = None
 	else:
 		SlimHTML = HTMLPagesList + ContentHTML
 	if not LightRun:
 		WriteFile(PagePath, HTML)
-	
+
+	#PrintPercentDots(ProcPercent)
 	return [File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image]
 
-#def MultiprocHandlePage(Data):
+def MultiprocHandlePage(d):
-#	pass
+	return HandlePage(d['Flags'], d['Page'], d['Pages'], d['Categories'], d['LimitFiles'], d['Snippets'], d['ConfMenu'], d['Locale'])
+
+def MultiprocPagePreprocessor(d):
+	return PagePreprocessor(d['Path'], d['TempPath'], d['Type'], d['Template'], d['SiteRoot'], d['GlobalMacros'], d['CategoryUncategorized'], d['LightRun'])
 
 def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale):
-	PagesPaths, PostsPaths, Pages, MadePages, MultiprocPages, Categories = [], [], [], [], [], {}
+	PagesPaths, PostsPaths, Pages, MadePages, Categories = [], [], [], [], {}
+	PoolSize = cpu_count()
 	OutDir, MarkdownExts, Sorting, MinifyKeepComments = Flags['OutDir'], Flags['MarkdownExts'], Flags['Sorting'], Flags['MinifyKeepComments']
 	SiteName, BlogName, SiteTagline = Flags['SiteName'], Flags['BlogName'], Flags['SiteTagline']
 	SiteTemplate, SiteLang = Flags['SiteTemplate'], Flags['SiteLang']
@@ -406,6 +412,7 @@ def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale):
 		PostsPaths.reverse()
 
 	print("[I] Preprocessing Source Pages")
+	MultiprocPages = []
 	for Type in ['Page', 'Post']:
 		if Type == 'Page':
 			Files = PagesPaths
@@ -416,10 +423,13 @@ def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale):
 		for File in Files:
 			TempPath = f"{PathPrefix}{File}"
 			LightRun = False if LimitFiles == False or TempPath in LimitFiles else True
-			Content, Titles, Meta = PagePreprocessor(f"{Type}s/{File}", Type, SiteTemplate, SiteRoot, GlobalMacros, CategoryUncategorized, LightRun=LightRun)
+			MultiprocPages += [{'Path':f"{Type}s/{File}", 'TempPath':TempPath, 'Type':Type, 'Template':SiteTemplate, 'SiteRoot':SiteRoot, 'GlobalMacros':GlobalMacros, 'CategoryUncategorized':CategoryUncategorized, 'LightRun':LightRun}]
-			Pages += [[TempPath, Content, Titles, Meta]]
+	with Pool(PoolSize) as MultiprocPool:
-			for Cat in Meta['Categories']:
+		Pages = MultiprocPool.map(MultiprocPagePreprocessor, MultiprocPages)
-				Categories.update({Cat:''})
+	#print() # Make newline after percentage dots
+	for File, Content, Titles, Meta in Pages:
+		for Cat in Meta['Categories']:
+			Categories.update({Cat:''})
 	PugCompileList(OutDir, Pages, LimitFiles)
 
 	if Categories:
@@ -458,11 +468,11 @@ def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale):
 				ConfMenu[i] = None
 
 	print("[I] Writing Pages")
+	MultiprocPages = []
 	for Page in Pages:
-		#print(f'-> {File}')
+		MultiprocPages += [{'Flags':Flags, 'Page':Page, 'Pages':Pages, 'Categories':Categories, 'LimitFiles':LimitFiles, 'Snippets':Snippets, 'ConfMenu':ConfMenu, 'Locale':Locale}]
-		MadePages += [HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, Locale)]
+	with Pool(PoolSize) as MultiprocPool:
-		#MultiprocPages += [{'Flags':Flags, 'Page':Page, 'Pages':Pages, 'Categories':Categories, 'LimitFiles':LimitFiles, 'Snippets':Snippets, 'ConfMenu':ConfMenu, 'Locale':Locale}]
+		MadePages = MultiprocPool.map(MultiprocHandlePage, MultiprocPages)
-	#with Pool(cpu_count()) as MultiprocPool:
+	#print() # Make newline after percentage dots
-		#MadePages = MultiprocPool.map(HandlePage, MultiprocPages)
 
 	return MadePages

Source/Modules/Utils.py

@@ -157,3 +157,11 @@ def LoadLocale(Lang):
 
 def IsLightRun(File, LimitFiles):
 	return False if LimitFiles == False or File in LimitFiles else True
+
+def PrintPercentDots(Percent):
+	Cur, Tot = Percent
+	CalcCur = int((Cur/Tot)*100)
+	if CalcCur % 10 == 0:
+		CalcPrev = int((Cur-1/Tot)*100)
+		if CalcPrev % 10 != 0:
+			print('.', end='')

TODO

@@ -1,6 +1,7 @@
+- Showing build percentage as dots
+- Custom thread number, 0 to disable multithreading
 - Posts in draft state (will not be compiled)
 - Check if external tools (pug-cli, html2gmi) are installed
-- Fix and optimize differential building
 - Static code syntax highlighing
 - Override internal HTML snippets (meta lines, page lists, ...) with config file in Templates/NAME.ini
 - Specify input folder(s)