From e28f75043c48f4b7497fce3e6bed30a53378bbbb Mon Sep 17 00:00:00 2001 From: octospacc Date: Wed, 31 Aug 2022 00:18:23 +0200 Subject: [PATCH] Multithreading yay --- README.md | 5 +++-- Source/Build.py | 8 ++++++-- Source/Modules/Site.py | 44 +++++++++++++++++++++++++---------------- Source/Modules/Utils.py | 8 ++++++++ TODO | 3 ++- 5 files changed, 46 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 2f79d95..c68899a 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,9 @@ Needed for Gemtext output support: ## Features roadmap -- [ ] Multithreading +- [ ] Showing percentage of build process +- [x] Multithreading +- [x] Differential building - [ ] Overriding internal HTML snippets for template-specific ones - [ ] Static syntax highlighing for code blocks in any page - [x] File name used as a title for pages without one @@ -65,7 +67,6 @@ Needed for Gemtext output support: - [x] Custom and automatic page sorting - [ ] SCSS compilation support for CSS templates - [ ] Pug support for base templates and page side parts -- [ ] Differential recompile (to optimize resource waste on non-ephemeral servers) - [ ] Hot-recompile (for website development) - [x] TXT sitemap generation - [x] Atom + RSS feed generation diff --git a/Source/Build.py b/Source/Build.py index 191116f..88340b9 100755 --- a/Source/Build.py +++ b/Source/Build.py @@ -120,6 +120,10 @@ def Main(Args, FeedEntries): CheckSafeOutDir(OutDir) print(f"[I] Outputting to: {OutDir}/") + Logging = Args.Logging + Threads = Args.Threads + DiffBuild = Args.DiffBuild + BlogName = Flags['BlogName'] = OptionChoose('', Args.BlogName, ReadConf(SiteConf, 'Site', 'BlogName')) SiteTagline = Flags['SiteTagline'] = OptionChoose('', Args.SiteTagline, ReadConf(SiteConf, 'Site', 'Tagline')) SiteTemplate = Flags['SiteTemplate'] = OptionChoose('Default.html', Args.SiteTemplate, ReadConf(SiteConf, 'Site', 'Template')) @@ -132,7 +136,6 @@ def Main(Args, FeedEntries): NoScripts = Flags['NoScripts'] = StringBoolChoose(False, Args.NoScripts, ReadConf(SiteConf, 'Site', 'NoScripts')) FolderRoots = Flags['FolderRoots'] = literal_eval(Args.FolderRoots) if Args.FolderRoots else {} - DiffBuild = Args.DiffBuild ActivityPubTypeFilter = Flags['ActivityPubTypeFilter'] = OptionChoose('Post', Args.ActivityPubTypeFilter, ReadConf(SiteConf, 'ActivityPub', 'TypeFilter')) ActivityPubHoursLimit = Flags['ActivityPubHoursLimit'] = OptionChoose(168, Args.ActivityPubHoursLimit, ReadConf(SiteConf, 'ActivityPub', 'HoursLimit')) @@ -250,6 +253,7 @@ if __name__ == '__main__': Parser = argparse.ArgumentParser() Parser.add_argument('--Logging', type=str) # Levels: Debug, Verbose, Info, Warning, Error. + Parser.add_argument('--Threads', type=str) Parser.add_argument('--DiffBuild', type=str) Parser.add_argument('--OutputDir', type=str) #Parser.add_argument('--InputDir', type=str) @@ -293,4 +297,4 @@ if __name__ == '__main__': Main( Args=Args, FeedEntries=FeedEntries) - print(f"[I] ✅ Done! ({round(time.process_time(),3)} / {round(time.time()-StartTime,3)} s)") + print(f"[I] ✅ Done! ({round(time.time()-StartTime,3)}s)") diff --git a/Source/Modules/Site.py b/Source/Modules/Site.py index 296aeb2..329e3b9 100644 --- a/Source/Modules/Site.py +++ b/Source/Modules/Site.py @@ -84,7 +84,7 @@ def TemplatePreprocessor(Text): Meta.update({i:MetaDefault[i]}) return Meta -def PagePreprocessor(Path, Type, SiteTemplate, SiteRoot, GlobalMacros, CategoryUncategorized, LightRun=False): +def PagePreprocessor(Path, TempPath, Type, SiteTemplate, SiteRoot, GlobalMacros, CategoryUncategorized, LightRun=False): File = ReadFile(Path) Path = Path.lower() Content, Titles, DashyTitles, HTMLTitlesFound, Macros, Meta, MetaDefault = '', [], [], False, '', '', { @@ -179,7 +179,8 @@ def PagePreprocessor(Path, Type, SiteTemplate, SiteRoot, GlobalMacros, CategoryU if GlobalMacros: Meta['Macros'].update(GlobalMacros) Meta['Macros'].update(ReadConf(LoadConfStr('[Macros]\n' + Macros), 'Macros')) - return Content, Titles, Meta + #PrintPercentDots(ProcPercent) + return [TempPath, Content, Titles, Meta] def PagePostprocessor(FileType, Text, Meta): for e in Meta['Macros']: @@ -336,7 +337,7 @@ def HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, L For='Menu', MarkdownExts=MarkdownExts, MenuStyle=TemplateMeta['MenuStyle']) - + HTML, ContentHTML, Description, Image = PatchHTML( File=File, HTML=TemplatesText[Meta['Template']], @@ -369,21 +370,26 @@ def HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, L if not LightRun: HTML = WriteImgAltAndTitle(HTML, ImgAltToTitle, ImgTitleToAlt) ContentHTML = WriteImgAltAndTitle(ContentHTML, ImgAltToTitle, ImgTitleToAlt) - + if LightRun: SlimHTML = None else: SlimHTML = HTMLPagesList + ContentHTML if not LightRun: WriteFile(PagePath, HTML) - + + #PrintPercentDots(ProcPercent) return [File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image] -#def MultiprocHandlePage(Data): -# pass +def MultiprocHandlePage(d): + return HandlePage(d['Flags'], d['Page'], d['Pages'], d['Categories'], d['LimitFiles'], d['Snippets'], d['ConfMenu'], d['Locale']) + +def MultiprocPagePreprocessor(d): + return PagePreprocessor(d['Path'], d['TempPath'], d['Type'], d['Template'], d['SiteRoot'], d['GlobalMacros'], d['CategoryUncategorized'], d['LightRun']) def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale): - PagesPaths, PostsPaths, Pages, MadePages, MultiprocPages, Categories = [], [], [], [], [], {} + PagesPaths, PostsPaths, Pages, MadePages, Categories = [], [], [], [], {} + PoolSize = cpu_count() OutDir, MarkdownExts, Sorting, MinifyKeepComments = Flags['OutDir'], Flags['MarkdownExts'], Flags['Sorting'], Flags['MinifyKeepComments'] SiteName, BlogName, SiteTagline = Flags['SiteName'], Flags['BlogName'], Flags['SiteTagline'] SiteTemplate, SiteLang = Flags['SiteTemplate'], Flags['SiteLang'] @@ -406,6 +412,7 @@ def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale): PostsPaths.reverse() print("[I] Preprocessing Source Pages") + MultiprocPages = [] for Type in ['Page', 'Post']: if Type == 'Page': Files = PagesPaths @@ -416,10 +423,13 @@ def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale): for File in Files: TempPath = f"{PathPrefix}{File}" LightRun = False if LimitFiles == False or TempPath in LimitFiles else True - Content, Titles, Meta = PagePreprocessor(f"{Type}s/{File}", Type, SiteTemplate, SiteRoot, GlobalMacros, CategoryUncategorized, LightRun=LightRun) - Pages += [[TempPath, Content, Titles, Meta]] - for Cat in Meta['Categories']: - Categories.update({Cat:''}) + MultiprocPages += [{'Path':f"{Type}s/{File}", 'TempPath':TempPath, 'Type':Type, 'Template':SiteTemplate, 'SiteRoot':SiteRoot, 'GlobalMacros':GlobalMacros, 'CategoryUncategorized':CategoryUncategorized, 'LightRun':LightRun}] + with Pool(PoolSize) as MultiprocPool: + Pages = MultiprocPool.map(MultiprocPagePreprocessor, MultiprocPages) + #print() # Make newline after percentage dots + for File, Content, Titles, Meta in Pages: + for Cat in Meta['Categories']: + Categories.update({Cat:''}) PugCompileList(OutDir, Pages, LimitFiles) if Categories: @@ -458,11 +468,11 @@ def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale): ConfMenu[i] = None print("[I] Writing Pages") + MultiprocPages = [] for Page in Pages: - #print(f'-> {File}') - MadePages += [HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, Locale)] - #MultiprocPages += [{'Flags':Flags, 'Page':Page, 'Pages':Pages, 'Categories':Categories, 'LimitFiles':LimitFiles, 'Snippets':Snippets, 'ConfMenu':ConfMenu, 'Locale':Locale}] - #with Pool(cpu_count()) as MultiprocPool: - #MadePages = MultiprocPool.map(HandlePage, MultiprocPages) + MultiprocPages += [{'Flags':Flags, 'Page':Page, 'Pages':Pages, 'Categories':Categories, 'LimitFiles':LimitFiles, 'Snippets':Snippets, 'ConfMenu':ConfMenu, 'Locale':Locale}] + with Pool(PoolSize) as MultiprocPool: + MadePages = MultiprocPool.map(MultiprocHandlePage, MultiprocPages) + #print() # Make newline after percentage dots return MadePages diff --git a/Source/Modules/Utils.py b/Source/Modules/Utils.py index 5bb9de2..21acf3c 100644 --- a/Source/Modules/Utils.py +++ b/Source/Modules/Utils.py @@ -157,3 +157,11 @@ def LoadLocale(Lang): def IsLightRun(File, LimitFiles): return False if LimitFiles == False or File in LimitFiles else True + +def PrintPercentDots(Percent): + Cur, Tot = Percent + CalcCur = int((Cur/Tot)*100) + if CalcCur % 10 == 0: + CalcPrev = int((Cur-1/Tot)*100) + if CalcPrev % 10 != 0: + print('.', end='') \ No newline at end of file diff --git a/TODO b/TODO index fe00689..9b0b283 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,7 @@ +- Showing build percentage as dots +- Custom thread number, 0 to disable multithreading - Posts in draft state (will not be compiled) - Check if external tools (pug-cli, html2gmi) are installed -- Fix and optimize differential building - Static code syntax highlighing - Override internal HTML snippets (meta lines, page lists, ...) with config file in Templates/NAME.ini - Specify input folder(s)