Minor code tweaks; Add: build time reporting, custom output path

This commit is contained in:
octospacc 2022-08-23 17:25:05 +02:00
parent 93a725839e
commit 342b2e8163
8 changed files with 63 additions and 95 deletions

View File

@ -28,19 +28,19 @@ from Modules.Markdown import *
from Modules.Site import *
from Modules.Sitemap import *
from Modules.Utils import *
def ResetPublic():
for i in ('public', 'public.gmi'):
def ResetPublic(OutputDir):
for i in (OutputDir, f"{OutputDir}.gmi"):
try:
shutil.rmtree(i)
except FileNotFoundError:
pass
def DelTmp():
def DelTmp(OutputDir):
for Ext in FileExtensions['Tmp']:
for File in Path('public').rglob(f"*.{Ext}"):
for File in Path(OutputDir).rglob(f"*.{Ext}"):
os.remove(File)
for Dir in ('public', 'public.gmi'):
for Dir in (OutputDir, f"{OutputDir}.gmi"):
for File in Path(Dir).rglob('*.tmp'):
os.remove(File)
@ -72,6 +72,10 @@ def Main(Args, FeedEntries):
HavePages, HavePosts = False, False
SiteConf = LoadConfFile('Site.ini')
OutputDir = Args.OutputDir if Args.OutputDir else ReadConf(SiteConf, 'Site', 'OutputDir') if ReadConf(SiteConf, 'Site', 'OutputDir') else 'public'
OutputDir = OutputDir.removesuffix('/')
print(f"[I] Outputting to {OutputDir}/")
SiteName = Args.SiteName if Args.SiteName else ReadConf(SiteConf, 'Site', 'Name') if ReadConf(SiteConf, 'Site', 'Name') else ''
BlogName = Args.BlogName if Args.BlogName else ReadConf(SiteConf, 'Site', 'BlogName') if ReadConf(SiteConf, 'Site', 'BlogName') else ''
SiteTagline = Args.SiteTagline if Args.SiteTagline else ReadConf(SiteConf, 'Site', 'Tagline') if ReadConf(SiteConf, 'Site', 'Tagline') else ''
@ -101,25 +105,26 @@ def Main(Args, FeedEntries):
else:
ConfMenu = []
ResetPublic()
ResetPublic(OutputDir)
if os.path.isdir('Pages'):
HavePages = True
shutil.copytree('Pages', 'public')
shutil.copytree('Pages', OutputDir)
if GemtextOut:
shutil.copytree('Pages', 'public.gmi', ignore=IgnoreFiles)
shutil.copytree('Pages', f"{OutputDir}.gmi", ignore=IgnoreFiles)
if os.path.isdir('Posts'):
HavePosts = True
shutil.copytree('Posts', 'public/Posts')
shutil.copytree('Posts', f"{OutputDir}/Posts")
if GemtextOut:
shutil.copytree('Posts', 'public.gmi/Posts', ignore=IgnoreFiles)
shutil.copytree('Posts', f"{OutputDir}.gmi/Posts", ignore=IgnoreFiles)
if not HavePages and not HavePosts:
if not (HavePages or HavePosts):
print("[E] No Pages or posts found. Nothing to do, exiting!")
exit(1)
print("[I] Generating HTML")
Pages = MakeSite(
OutputDir=OutputDir,
TemplatesText=LoadFromDir('Templates', ['*.htm', '*.html']),
StaticPartsText=LoadFromDir('StaticParts', ['*.htm', '*.html']),
DynamicParts=DynamicParts,
@ -146,6 +151,7 @@ def Main(Args, FeedEntries):
print("[I] Generating Feeds")
for FeedType in (True, False):
MakeFeed(
OutputDir=OutputDir,
CategoryFilter=FeedCategoryFilter,
Pages=Pages,
SiteName=SiteName,
@ -158,7 +164,7 @@ def Main(Args, FeedEntries):
if SitemapOut:
print("[I] Generating Sitemap")
MakeSitemap(Pages, SiteDomain)
MakeSitemap(OutputDir, Pages, SiteDomain)
if ActivityPub and MastodonURL and MastodonToken and SiteDomain:
print("[I] Mastodon Stuff")
@ -176,11 +182,11 @@ def Main(Args, FeedEntries):
MastodonPosts = []
for File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image in Pages:
File = 'public/{}.html'.format(StripExt(File))
File = f"{OutputDir}/{StripExt(File)}.html"
Content = ReadFile(File)
Post = ''
for p in MastodonPosts:
if p['Link'] == SiteDomain + '/' + File[len('public/'):]:
if p['Link'] == SiteDomain + '/' + File[len(f"{OutputDir}/"):]:
Post = '<br><h3>{StrComments}</h3><a href="{URL}" rel="noopener" target="_blank">{StrOpen} ↗️</a>'.format(
StrComments=Locale['Comments'],
StrOpen=Locale['OpenInNewTab'],
@ -192,19 +198,21 @@ def Main(Args, FeedEntries):
if GemtextOut:
print("[I] Generating Gemtext")
GemtextCompileList(
OutputDir,
Pages,
Header=Args.GemtextHeader if Args.GemtextHeader else f"# {SiteName}\n\n" if SiteName else '')
print("[I] Cleaning Temporary Files")
DelTmp()
DelTmp(OutputDir)
print("[I] Copying Assets")
os.system("cp -R Assets/* public/")
os.system(f"cp -R Assets/* {OutputDir}/")
print("[I] Done!")
if __name__ == '__main__':
Parser = argparse.ArgumentParser()
Parser.add_argument('--OutputDir', type=str)
Parser.add_argument('--Sorting', type=str)
Parser.add_argument('--SiteLang', type=str)
Parser.add_argument('--SiteRoot', type=str)

View File

@ -62,7 +62,7 @@ def MastodonShare(InstanceURL, Token, TypeFilter, CategoryFilter, HoursLimit, Pa
Parse = BeautifulSoup(ContentHTML, 'html.parser')
Paragraphs = Parse.p.get_text().split('\n')
Read = '...' + Locale['ReadFullPost'] + ':\n'
URL = '{}/{}.html'.format(SiteDomain, StripExt(File))
URL = f"{SiteDomain}/{StripExt(File)}.html"
for p in Paragraphs:
if p and len(Read+Desc+p)+25 < 500:
Desc += p + '\n\n'

View File

@ -12,7 +12,7 @@
from Libs.feedgen.feed import FeedGenerator
from Modules.Utils import *
def MakeFeed(CategoryFilter, Pages, SiteName, SiteTagline, SiteDomain, MaxEntries, Lang, FullSite=False, Minify=False):
def MakeFeed(OutputDir, CategoryFilter, Pages, SiteName, SiteTagline, SiteDomain, MaxEntries, Lang, FullSite=False, Minify=False):
Feed = FeedGenerator()
Link = SiteDomain if SiteDomain else ' '
Feed.id(Link)
@ -34,8 +34,8 @@ def MakeFeed(CategoryFilter, Pages, SiteName, SiteTagline, SiteDomain, MaxEntrie
if FullSite or (not FullSite and Meta['Type'] == 'Post' and (not CategoryFilter or (CategoryFilter and (CategoryFilter in Meta['Categories'] or CategoryFilter == '*')))):
Entry = Feed.add_entry()
FileName = File.split('/')[-1]
File = '{}.html'.format(StripExt(File))
Content = ReadFile('public/'+File)
File = f"{StripExt(File)}.html"
Content = ReadFile(f"{OutputDir}/{File}")
Link = SiteDomain + '/' + File if SiteDomain else ' '
CreatedOn = GetFullDate(Meta['CreatedOn'])
EditedOn = GetFullDate(Meta['EditedOn'])
@ -51,11 +51,8 @@ def MakeFeed(CategoryFilter, Pages, SiteName, SiteTagline, SiteDomain, MaxEntrie
EditedOn = EditedOn if EditedOn else CreatedOn if CreatedOn and not EditedOn else '1970-01-01T00:00+00:00'
Entry.updated(EditedOn)
if not os.path.exists('public/feed'):
os.mkdir('public/feed')
if FullSite:
FeedType = 'site.'
else:
FeedType = ''
Feed.atom_file('public/feed/' + FeedType + 'atom.xml', pretty=(not Minify))
Feed.rss_file('public/feed/' + FeedType + 'rss.xml', pretty=(not Minify))
if not os.path.exists(f"{OutputDir}/feed"):
os.mkdir(f"{OutputDir}/feed")
FeedType = 'site.' if FullSite else ''
Feed.atom_file(f"{OutputDir}/feed/{FeedType}atom.xml", pretty=(not Minify))
Feed.rss_file(f"{OutputDir}/feed/{FeedType}rss.xml", pretty=(not Minify))

View File

@ -22,20 +22,20 @@ def FixGemlogDateLine(Line):
Line = Words[0] + '\n' + Words[1][1:] + ' ' + ' '.join(Words[2:])
return Line
def GemtextCompileList(Pages, Header=''):
def GemtextCompileList(OutputDir, Pages, Header=''):
Cmd = ''
for File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image in Pages:
Src = 'public.gmi/{}.html.tmp'.format(StripExt(File))
Dst = 'public.gmi/{}.gmi'.format(StripExt(File))
Src = f"{OutputDir}.gmi/{StripExt(File)}.html.tmp"
Dst = f"{OutputDir}.gmi/{StripExt(File)}.gmi"
SlimHTML = StripAttrs(SlimHTML)
for i in ('ol', 'ul', 'li'):
for j in ('<'+i+'>', '</'+i+'>'):
SlimHTML = SlimHTML.replace(j, '')
WriteFile(Src, SlimHTML.replace('</a>', '</a><br>').replace('.html', '.gmi')) # TODO: Adjust links properly..
Cmd += 'cat "{}" | html2gmi > "{}"; '.format(Src, Dst)
Cmd += f'cat "{Src}" | html2gmi > "{Dst}"; '
os.system(Cmd)
for File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image in Pages:
Dst = 'public.gmi/{}.gmi'.format(StripExt(File))
Dst = f"{OutputDir}.gmi/{StripExt(File)}.gmi"
Gemtext = ''
for Line in ReadFile(Dst).splitlines():
Line = FixGemlogDateLine(Line)
@ -52,43 +52,3 @@ def ParseTag(Content):
print(Content)
Parse = BeautifulSoup(str(Content), 'html.parser')
Tag = Parse.find()
"""
def HTML2Gemtext(Pages, SiteName, SiteTagline):
#os.mkdir('public.gmi')
for File, Content, Titles, Meta, HTMLContent, Description, Image in Pages:
Gemtext = ''
Content = HTMLContent
print(File)
while len(Content) != 0:
BlockStart = Content.find('<')
TagEnd = Content.find('>')
Parse = BeautifulSoup(Content, 'html.parser')
Tag = Parse.find()
#if Tag.name in ('a'):
# if 'href' in Tag.attrs:
# pass
for i in Tag.contents:
ParseTag(i)
if Tag.name in ('h1', 'h2', 'h3'):
Gemtext += '#' * int(Tag.name[1]) + ' '
elif Tag.name in ('h4', 'h5', 'h6'):
Gemtext += '### '
elif Tag.name in ('li'):
Gemtext += '* '
Gemtext += str(Tag.get_text()) + '\n\n'
#print(File, Tag.name, len(Tag.contents))
if Tag.name in ClosedTags:
Str = '</{}>'.format(Tag.name)
elif Tag.name in OpenTags:
Str = '>'
BlockEnd = Content.find(Str) + len(Str)
Content = Content.replace(Content[BlockStart:TagEnd], '').replace(Content[BlockEnd-len(Str):BlockEnd], '')
#print(BlockStart, TagEnd, BlockEnd, Tag.contents)
#print(Content[BlockStart:BlockEnd])
#Gemtext += Content[BlockStart:BlockEnd]
Content = Content[BlockEnd:]
PagePath = 'public.gmi/{}.gmi'.format(StripExt(File))
WriteFile(PagePath, Gemtext)
#exit()
"""

View File

@ -12,13 +12,13 @@
import os
from Modules.Utils import *
def PugCompileList(Pages):
def PugCompileList(OutputDir, Pages):
# Pug-cli seems to shit itself with folder paths as input, so we pass ALL the files as arguments
Paths = ''
for File, Content, Titles, Meta in Pages:
if File.lower().endswith('.pug'):
Path = 'public/{}'.format(File)
Path = f'{OutputDir}/{File}'
WriteFile(Path, Content)
Paths += '"{}" '.format(Path)
Paths += f'"{Path}" '
if Paths:
os.system('pug -P {} > /dev/null'.format(Paths))
os.system(f'pug -P {Paths} > /dev/null')

View File

@ -7,6 +7,7 @@
| Copyright (C) 2022, OctoSpacc |
| ================================= """
from datetime import datetime
from Libs import htmlmin
from Libs.bs4 import BeautifulSoup
from Modules.Config import *
@ -21,12 +22,12 @@ def DashifyTitle(Title, Done=[]):
def MakeLinkableTitle(Line, Title, DashTitle, Type):
if Type == 'md':
Index = Title.split(' ')[0].count('#')
return '<h{} id="{}">{}</h{}>'.format(Index, DashTitle, Title[Index+1:], Index)
return f'<h{Index} id="{DashTitle}">{Title[Index+1:]}</h{Index}>'
elif Type == 'pug':
NewLine = ''
Index = Line.find('h')
NewLine += Line[:Index]
NewLine += "{}(id='{}')".format(Line[Index:Index+2], DashTitle)
NewLine += f"{Line[Index:Index+2]}(id='{DashTitle}')"
NewLine += Line[Index+2:]
return NewLine
@ -59,16 +60,16 @@ def MakeContentHeader(Meta, Locale, Categories=''):
Header = ''
for i in ['CreatedOn', 'EditedOn']:
if Meta[i]:
Header += '{} {} \n'.format(Locale[i], Meta[i])
Header += f"{Locale[i]}: {Meta[i]} \n"
if Categories:
Header += '{}: {} \n'.format(Locale['Categories'], Categories)
Header += f"{Locale['Categories']}: {Categories} \n"
return markdown(Header.rstrip())
def MakeCategoryLine(File, Meta):
Categories = ''
if Meta['Categories']:
for i in Meta['Categories']:
Categories += ' [{}]({}{}.html) '.format(i, GetPathLevels(File) + 'Categories/', i)
Categories += f" [{i}]({GetPathLevels(File)}Categories/{i}.html) "
return Categories
def GetHTMLPagesList(Pages, BlogName, SiteRoot, PathPrefix, Unite=[], Type='Page', Category=None, For='Menu', MarkdownExts=(), MenuStyle='Default'):
@ -240,7 +241,7 @@ def MakeListTitle(File, Meta, Titles, Prefer, SiteRoot, BlogName, PathPrefix='')
'{}{}.html'.format(PathPrefix, StripExt(File)))
if Meta['Type'] == 'Post':
CreatedOn = Meta['CreatedOn'] if Meta['CreatedOn'] else '?'
Title = '[{}] {}'.format(CreatedOn, Title)
Title = f"[{CreatedOn}] {Title}"
return Title
def FormatTitles(Titles, Flatten=False):
@ -252,7 +253,7 @@ def FormatTitles(Titles, Flatten=False):
Title = t.lstrip('#')
DashyTitle = DashifyTitle(Title, DashyTitles)
DashyTitles += [DashyTitle]
Title = '[{}](#{})'.format(Title, DashyTitle)
Title = f"[{Title}](#{DashyTitle})"
MDTitles += Heading + Title + '\n'
return markdown(MDTitles)
@ -329,6 +330,7 @@ def PatchHTML(File, HTML, StaticPartsText, DynamicParts, DynamicPartsText, HTMLP
HTML = ReplWithEsc(HTML, '[staticoso:Page:Style]', Meta['Style'])
HTML = ReplWithEsc(HTML, '[staticoso:Page:Content]', Content)
HTML = ReplWithEsc(HTML, '[staticoso:Page:ContentInfo]', MakeContentHeader(Meta, Locale, MakeCategoryLine(File, Meta)))
HTML = ReplWithEsc(HTML, '[staticoso:BuildTime]', datetime.now().strftime('%Y-%m-%d %H:%M'))
HTML = ReplWithEsc(HTML, '[staticoso:Site:Name]', SiteName)
HTML = ReplWithEsc(HTML, '[staticoso:Site:AbsoluteRoot]', SiteRoot)
HTML = ReplWithEsc(HTML, '[staticoso:Site:RelativeRoot]', GetPathLevels(PagePath))
@ -365,7 +367,7 @@ def DoMinifyHTML(HTML):
convert_charrefs=True,
keep_pre=True)
def MakeSite(TemplatesText, StaticPartsText, DynamicParts, DynamicPartsText, ConfMenu, GlobalMacros, SiteName, BlogName, SiteTagline, SiteTemplate, SiteDomain, SiteRoot, FolderRoots, SiteLang, Locale, Minify, NoScripts, ImgAltAndTitle, Sorting, MarkdownExts, AutoCategories):
def MakeSite(OutputDir, TemplatesText, StaticPartsText, DynamicParts, DynamicPartsText, ConfMenu, GlobalMacros, SiteName, BlogName, SiteTagline, SiteTemplate, SiteDomain, SiteRoot, FolderRoots, SiteLang, Locale, Minify, NoScripts, ImgAltAndTitle, Sorting, MarkdownExts, AutoCategories):
PagesPaths, PostsPaths, Pages, MadePages, Categories = [], [], [], [], {}
for Ext in FileExtensions['Pages']:
for File in Path('Pages').rglob(f"*.{Ext}"):
@ -395,7 +397,7 @@ def MakeSite(TemplatesText, StaticPartsText, DynamicParts, DynamicPartsText, Con
Pages += [[File, Content, Titles, Meta]]
for Cat in Meta['Categories']:
Categories.update({Cat:''})
PugCompileList(Pages)
PugCompileList(OutputDir, Pages)
if Categories:
print("[I] Generating Category Lists")
@ -413,7 +415,7 @@ def MakeSite(TemplatesText, StaticPartsText, DynamicParts, DynamicPartsText, Con
MenuStyle='Flat')
if AutoCategories:
Dir = 'public/Categories'
Dir = f"{OutputDir}/Categories"
for Cat in Categories:
Exists = False
for File in Path(Dir).rglob(str(Cat)+'.*'):
@ -421,7 +423,7 @@ def MakeSite(TemplatesText, StaticPartsText, DynamicParts, DynamicPartsText, Con
break
if not Exists:
File = f"Categories/{Cat}.md"
FilePath = f"public/{File}"
FilePath = f"{OutputDir}/{File}"
WriteFile(FilePath, f"""\
// Title: {Cat}
// Type: Page
@ -442,7 +444,7 @@ def MakeSite(TemplatesText, StaticPartsText, DynamicParts, DynamicPartsText, Con
print("[I] Writing Pages")
for File, Content, Titles, Meta in Pages:
PagePath = 'public/{}.html'.format(StripExt(File))
PagePath = f"{OutputDir}/{StripExt(File)}.html"
if File.lower().endswith(FileExtensions['Markdown']):
Content = markdown(PagePostprocessor('md', Content, Meta), extensions=MarkdownExts)
elif File.lower().endswith(('.pug')):
@ -471,7 +473,7 @@ def MakeSite(TemplatesText, StaticPartsText, DynamicParts, DynamicPartsText, Con
DynamicParts=DynamicParts,
DynamicPartsText=DynamicPartsText,
HTMLPagesList=HTMLPagesList,
PagePath=PagePath[len('public/'):],
PagePath=PagePath[len(f"{OutputDir}/"):],
Content=Content,
Titles=Titles,
Meta=Meta,

View File

@ -10,10 +10,10 @@
from urllib.parse import quote as URLEncode
from Modules.Utils import *
def MakeSitemap(Pages, SiteDomain=''):
def MakeSitemap(OutputDir, Pages, SiteDomain=''):
Map = ''
Domain = SiteDomain + '/' if SiteDomain else ''
for File, Content, Titles, Meta, ContentHTML, SlimHTML, Description, Image in Pages:
File = '{}.html'.format(StripExt(File))
Domain = SiteDomain + '/' if SiteDomain else ' '
File = f"{StripExt(File)}.html"
Map += Domain + URLEncode(File) + '\n'
WriteFile('public/sitemap.txt', Map)
WriteFile(f"{OutputDir}/sitemap.txt", Map)

1
TODO
View File

@ -1,6 +1,7 @@
- Handle file extensions with any case sensitivity, not just lowercase; currently the bulk of the issue is finding the files on disk
- Test sorting by date for files not starting with date, and dated folders
- Fix arguments - some are only callable from CLI and not Site.ini - make them coherent with INI categories
- Accept Macros as CLI arguments
- Deprecate FolderRoots (Macros make it redundant)
- Fix ordering menu in Site.ini (not working for inner pages)
- Fix Python-Markdown is installed problem (to load our modules)