mirror of https://gitlab.com/octtspacc/staticoso
Multithreading yay
This commit is contained in:
parent
087c6c6f9a
commit
e28f75043c
|
@ -40,7 +40,9 @@ Needed for Gemtext output support:
|
|||
|
||||
## Features roadmap
|
||||
|
||||
- [ ] Multithreading
|
||||
- [ ] Showing percentage of build process
|
||||
- [x] Multithreading
|
||||
- [x] Differential building
|
||||
- [ ] Overriding internal HTML snippets for template-specific ones
|
||||
- [ ] Static syntax highlighing for code blocks in any page
|
||||
- [x] File name used as a title for pages without one
|
||||
|
@ -65,7 +67,6 @@ Needed for Gemtext output support:
|
|||
- [x] Custom and automatic page sorting
|
||||
- [ ] SCSS compilation support for CSS templates
|
||||
- [ ] Pug support for base templates and page side parts
|
||||
- [ ] Differential recompile (to optimize resource waste on non-ephemeral servers)
|
||||
- [ ] Hot-recompile (for website development)
|
||||
- [x] TXT sitemap generation
|
||||
- [x] Atom + RSS feed generation
|
||||
|
|
|
@ -120,6 +120,10 @@ def Main(Args, FeedEntries):
|
|||
CheckSafeOutDir(OutDir)
|
||||
print(f"[I] Outputting to: {OutDir}/")
|
||||
|
||||
Logging = Args.Logging
|
||||
Threads = Args.Threads
|
||||
DiffBuild = Args.DiffBuild
|
||||
|
||||
BlogName = Flags['BlogName'] = OptionChoose('', Args.BlogName, ReadConf(SiteConf, 'Site', 'BlogName'))
|
||||
SiteTagline = Flags['SiteTagline'] = OptionChoose('', Args.SiteTagline, ReadConf(SiteConf, 'Site', 'Tagline'))
|
||||
SiteTemplate = Flags['SiteTemplate'] = OptionChoose('Default.html', Args.SiteTemplate, ReadConf(SiteConf, 'Site', 'Template'))
|
||||
|
@ -132,7 +136,6 @@ def Main(Args, FeedEntries):
|
|||
|
||||
NoScripts = Flags['NoScripts'] = StringBoolChoose(False, Args.NoScripts, ReadConf(SiteConf, 'Site', 'NoScripts'))
|
||||
FolderRoots = Flags['FolderRoots'] = literal_eval(Args.FolderRoots) if Args.FolderRoots else {}
|
||||
DiffBuild = Args.DiffBuild
|
||||
|
||||
ActivityPubTypeFilter = Flags['ActivityPubTypeFilter'] = OptionChoose('Post', Args.ActivityPubTypeFilter, ReadConf(SiteConf, 'ActivityPub', 'TypeFilter'))
|
||||
ActivityPubHoursLimit = Flags['ActivityPubHoursLimit'] = OptionChoose(168, Args.ActivityPubHoursLimit, ReadConf(SiteConf, 'ActivityPub', 'HoursLimit'))
|
||||
|
@ -250,6 +253,7 @@ if __name__ == '__main__':
|
|||
|
||||
Parser = argparse.ArgumentParser()
|
||||
Parser.add_argument('--Logging', type=str) # Levels: Debug, Verbose, Info, Warning, Error.
|
||||
Parser.add_argument('--Threads', type=str)
|
||||
Parser.add_argument('--DiffBuild', type=str)
|
||||
Parser.add_argument('--OutputDir', type=str)
|
||||
#Parser.add_argument('--InputDir', type=str)
|
||||
|
@ -293,4 +297,4 @@ if __name__ == '__main__':
|
|||
Main(
|
||||
Args=Args,
|
||||
FeedEntries=FeedEntries)
|
||||
print(f"[I] ✅ Done! ({round(time.process_time(),3)} / {round(time.time()-StartTime,3)} s)")
|
||||
print(f"[I] ✅ Done! ({round(time.time()-StartTime,3)}s)")
|
||||
|
|
|
@ -84,7 +84,7 @@ def TemplatePreprocessor(Text):
|
|||
Meta.update({i:MetaDefault[i]})
|
||||
return Meta
|
||||
|
||||
def PagePreprocessor(Path, Type, SiteTemplate, SiteRoot, GlobalMacros, CategoryUncategorized, LightRun=False):
|
||||
def PagePreprocessor(Path, TempPath, Type, SiteTemplate, SiteRoot, GlobalMacros, CategoryUncategorized, LightRun=False):
|
||||
File = ReadFile(Path)
|
||||
Path = Path.lower()
|
||||
Content, Titles, DashyTitles, HTMLTitlesFound, Macros, Meta, MetaDefault = '', [], [], False, '', '', {
|
||||
|
@ -179,7 +179,8 @@ def PagePreprocessor(Path, Type, SiteTemplate, SiteRoot, GlobalMacros, CategoryU
|
|||
if GlobalMacros:
|
||||
Meta['Macros'].update(GlobalMacros)
|
||||
Meta['Macros'].update(ReadConf(LoadConfStr('[Macros]\n' + Macros), 'Macros'))
|
||||
return Content, Titles, Meta
|
||||
#PrintPercentDots(ProcPercent)
|
||||
return [TempPath, Content, Titles, Meta]
|
||||
|
||||
def PagePostprocessor(FileType, Text, Meta):
|
||||
for e in Meta['Macros']:
|
||||
|
@ -336,7 +337,7 @@ def HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, L
|
|||
For='Menu',
|
||||
MarkdownExts=MarkdownExts,
|
||||
MenuStyle=TemplateMeta['MenuStyle'])
|
||||
|
||||
|
||||
HTML, ContentHTML, Description, Image = PatchHTML(
|
||||
File=File,
|
||||
HTML=TemplatesText[Meta['Template']],
|
||||
|
@ -369,21 +370,26 @@ def HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, L
|
|||
if not LightRun:
|
||||
HTML = WriteImgAltAndTitle(HTML, ImgAltToTitle, ImgTitleToAlt)
|
||||
ContentHTML = WriteImgAltAndTitle(ContentHTML, ImgAltToTitle, ImgTitleToAlt)
|
||||
|
||||
|
||||
if LightRun:
|
||||
SlimHTML = None
|
||||
else:
|
||||
SlimHTML = HTMLPagesList + ContentHTML
|
||||
if not LightRun:
|
||||
WriteFile(PagePath, HTML)
|
||||
|
||||
|
||||
#PrintPercentDots(ProcPercent)
|
||||
return [File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image]
|
||||
|
||||
#def MultiprocHandlePage(Data):
|
||||
# pass
|
||||
def MultiprocHandlePage(d):
|
||||
return HandlePage(d['Flags'], d['Page'], d['Pages'], d['Categories'], d['LimitFiles'], d['Snippets'], d['ConfMenu'], d['Locale'])
|
||||
|
||||
def MultiprocPagePreprocessor(d):
|
||||
return PagePreprocessor(d['Path'], d['TempPath'], d['Type'], d['Template'], d['SiteRoot'], d['GlobalMacros'], d['CategoryUncategorized'], d['LightRun'])
|
||||
|
||||
def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale):
|
||||
PagesPaths, PostsPaths, Pages, MadePages, MultiprocPages, Categories = [], [], [], [], [], {}
|
||||
PagesPaths, PostsPaths, Pages, MadePages, Categories = [], [], [], [], {}
|
||||
PoolSize = cpu_count()
|
||||
OutDir, MarkdownExts, Sorting, MinifyKeepComments = Flags['OutDir'], Flags['MarkdownExts'], Flags['Sorting'], Flags['MinifyKeepComments']
|
||||
SiteName, BlogName, SiteTagline = Flags['SiteName'], Flags['BlogName'], Flags['SiteTagline']
|
||||
SiteTemplate, SiteLang = Flags['SiteTemplate'], Flags['SiteLang']
|
||||
|
@ -406,6 +412,7 @@ def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale):
|
|||
PostsPaths.reverse()
|
||||
|
||||
print("[I] Preprocessing Source Pages")
|
||||
MultiprocPages = []
|
||||
for Type in ['Page', 'Post']:
|
||||
if Type == 'Page':
|
||||
Files = PagesPaths
|
||||
|
@ -416,10 +423,13 @@ def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale):
|
|||
for File in Files:
|
||||
TempPath = f"{PathPrefix}{File}"
|
||||
LightRun = False if LimitFiles == False or TempPath in LimitFiles else True
|
||||
Content, Titles, Meta = PagePreprocessor(f"{Type}s/{File}", Type, SiteTemplate, SiteRoot, GlobalMacros, CategoryUncategorized, LightRun=LightRun)
|
||||
Pages += [[TempPath, Content, Titles, Meta]]
|
||||
for Cat in Meta['Categories']:
|
||||
Categories.update({Cat:''})
|
||||
MultiprocPages += [{'Path':f"{Type}s/{File}", 'TempPath':TempPath, 'Type':Type, 'Template':SiteTemplate, 'SiteRoot':SiteRoot, 'GlobalMacros':GlobalMacros, 'CategoryUncategorized':CategoryUncategorized, 'LightRun':LightRun}]
|
||||
with Pool(PoolSize) as MultiprocPool:
|
||||
Pages = MultiprocPool.map(MultiprocPagePreprocessor, MultiprocPages)
|
||||
#print() # Make newline after percentage dots
|
||||
for File, Content, Titles, Meta in Pages:
|
||||
for Cat in Meta['Categories']:
|
||||
Categories.update({Cat:''})
|
||||
PugCompileList(OutDir, Pages, LimitFiles)
|
||||
|
||||
if Categories:
|
||||
|
@ -458,11 +468,11 @@ def MakeSite(Flags, LimitFiles, Snippets, ConfMenu, GlobalMacros, Locale):
|
|||
ConfMenu[i] = None
|
||||
|
||||
print("[I] Writing Pages")
|
||||
MultiprocPages = []
|
||||
for Page in Pages:
|
||||
#print(f'-> {File}')
|
||||
MadePages += [HandlePage(Flags, Page, Pages, Categories, LimitFiles, Snippets, ConfMenu, Locale)]
|
||||
#MultiprocPages += [{'Flags':Flags, 'Page':Page, 'Pages':Pages, 'Categories':Categories, 'LimitFiles':LimitFiles, 'Snippets':Snippets, 'ConfMenu':ConfMenu, 'Locale':Locale}]
|
||||
#with Pool(cpu_count()) as MultiprocPool:
|
||||
#MadePages = MultiprocPool.map(HandlePage, MultiprocPages)
|
||||
MultiprocPages += [{'Flags':Flags, 'Page':Page, 'Pages':Pages, 'Categories':Categories, 'LimitFiles':LimitFiles, 'Snippets':Snippets, 'ConfMenu':ConfMenu, 'Locale':Locale}]
|
||||
with Pool(PoolSize) as MultiprocPool:
|
||||
MadePages = MultiprocPool.map(MultiprocHandlePage, MultiprocPages)
|
||||
#print() # Make newline after percentage dots
|
||||
|
||||
return MadePages
|
||||
|
|
|
@ -157,3 +157,11 @@ def LoadLocale(Lang):
|
|||
|
||||
def IsLightRun(File, LimitFiles):
|
||||
return False if LimitFiles == False or File in LimitFiles else True
|
||||
|
||||
def PrintPercentDots(Percent):
|
||||
Cur, Tot = Percent
|
||||
CalcCur = int((Cur/Tot)*100)
|
||||
if CalcCur % 10 == 0:
|
||||
CalcPrev = int((Cur-1/Tot)*100)
|
||||
if CalcPrev % 10 != 0:
|
||||
print('.', end='')
|
3
TODO
3
TODO
|
@ -1,6 +1,7 @@
|
|||
- Showing build percentage as dots
|
||||
- Custom thread number, 0 to disable multithreading
|
||||
- Posts in draft state (will not be compiled)
|
||||
- Check if external tools (pug-cli, html2gmi) are installed
|
||||
- Fix and optimize differential building
|
||||
- Static code syntax highlighing
|
||||
- Override internal HTML snippets (meta lines, page lists, ...) with config file in Templates/NAME.ini
|
||||
- Specify input folder(s)
|
||||
|
|
Loading…
Reference in New Issue