From 0567297de69f3d9a48ba37ea12b429edf2982838 Mon Sep 17 00:00:00 2001 From: octospacc Date: Mon, 27 Feb 2023 18:11:16 +0100 Subject: [PATCH] Add Sites/Main; Upd. build scripts; Refactoring (Site.py); Some work on transclusions --- App/Source/Modules/Site.py | 181 +++++++++++++------------- App/Source/Modules/Utils.py | 32 ++++- README.md | 1 + Sites/BuildAll.sh | 8 +- Sites/{Docs => }/LICENSE.md | 0 Sites/Main/.github/workflows/main.yml | 33 +++++ Sites/Main/.gitlab-ci.yml | 16 +++ Sites/Main/Pages/index.md | 16 +++ Sites/Main/Site.ini | 2 + Sites/Main/Templates/Default.html | 21 +++ 10 files changed, 210 insertions(+), 100 deletions(-) rename Sites/{Docs => }/LICENSE.md (100%) create mode 100644 Sites/Main/.github/workflows/main.yml create mode 100644 Sites/Main/.gitlab-ci.yml create mode 100644 Sites/Main/Pages/index.md create mode 100644 Sites/Main/Site.ini create mode 100644 Sites/Main/Templates/Default.html diff --git a/App/Source/Modules/Site.py b/App/Source/Modules/Site.py index 2603bc9..766a8d5 100644 --- a/App/Source/Modules/Site.py +++ b/App/Source/Modules/Site.py @@ -19,10 +19,62 @@ from Modules.Meta import * from Modules.Pug import * from Modules.Utils import * -def PatchHTML(Flags:dict, File, HTML:str, Snippets:dict, HTMLPagesList:str, PagePath:str, Content:str, Titles:list, Meta:dict, Categories, Locale:dict, LightRun): - f = NameSpace(Flags) +def HandleStaticParts(Html:str, Snippets:dict): + for e in Snippets['StaticParts']: + Html = ReplWithEsc(Html, f"[staticoso:StaticPart:{e}]", Snippets['StaticParts'][e]) + Html = ReplWithEsc(Html, f"", Snippets['StaticParts'][e]) + return Html - HTMLTitles = FormatTitles(Titles) +def HandleDynamicParts(Flags:dict, Html:str, Snippets:dict): + f = NameSpace(Flags) + Key = 'staticoso:dynamicpart' + if f'{Key}:' in Html.lower(): # Reduce unnecessary cycles + for Line in Html.splitlines(): + Line = Line.lstrip().rstrip() + LineLow = Line.lower() + if (LineLow.startswith(f'[{Key}:') and LineLow.endswith(']')) or (LineLow.startswith(f'<{Key}:') and LineLow.endswith('>')): + Path = Line[len(f'<{Key}:'):-1] + Section = Path.split('/')[-1] + if Section in f.DynamicParts: + Parts = f.DynamicParts[Section] + Text = '' + Parts = SureList(Parts) + for Part in Parts: + Text += Snippets['DynamicParts'][f'{Path}/{Part}'] + '\n' + else: + Text = '' + Html = ReplWithEsc(Html, f'[staticoso:DynamicPart:{Path}]', Text) + Html = ReplWithEsc(Html, f'', Text) + return Html + +# TODO: This would need to be handled either fully before or fully after after all pages' content has been transformed to HTML, else other markups end up in HTML and the page is broken +def HandleTransclusions(Html:str, Caller:str, Pages:list): + #if Type == 'Evals': # [% cmd %] | {% cmd %} + Targets = [] + Finding = Html + Start = Finding.find('{{') + while Start != -1: + Start = Start + 2 + Finding = Finding[Start:] + Stop = Finding.find('}}') + if Stop != -1: + Targets += [Finding[:Stop]] + Start = Finding.find('{{') + for Target in Targets: + # Maybe we should show an error message on possible recursive transclusion, as currently this doesn't handle escaped tokens + if Target != Caller: + for File, Content, _, _ in Pages: + if File == Target: + Html = ReplWithEsc(Html, '{{' + Target + '}}', Content) + break + return Html + +def PatchHtml(Flags:dict, Pages:list, Page:dict, Context:dict, Snippets:dict, Locale:dict, LightRun): + f = NameSpace(Flags) + File, PagePath, Content, Titles, Meta = tuple(Page.values()) + Html, HtmlPagesList, Categories = tuple(Context.values()) + + HtmlTitles = FormatTitles(Titles) BodyDescription, BodyImage = '', '' if not File.lower().endswith('.txt'): Soup = MkSoup(Content) @@ -59,50 +111,29 @@ def PatchHTML(Flags:dict, File, HTML:str, Snippets:dict, HTMLPagesList:str, Page TimeNow = datetime.now().strftime('%Y-%m-%d %H:%M') RelativeRoot = GetPathLevels(PagePath) - if 'staticoso:DynamicPart:' in HTML: # Reduce risk of unnecessary cycles - for Line in HTML.splitlines(): - Line = Line.lstrip().rstrip() - if (Line.startswith('[staticoso:DynamicPart:') and Line.endswith(']')) or (Line.startswith('')): - Path = Line[len('", Text) - - for i in range(2): - for e in Snippets['StaticParts']: - HTML = ReplWithEsc(HTML, f"[staticoso:StaticPart:{e}]", Snippets['StaticParts'][e]) - HTML = ReplWithEsc(HTML, f"", Snippets['StaticParts'][e]) + Html = WhileFuncResultChanges(HandleDynamicParts, {"Flags": Flags, "Html": Html, "Snippets": Snippets}, 'Html') + Html = WhileFuncResultChanges(HandleStaticParts, {"Html": Html, "Snippets": Snippets}, 'Html') if LightRun: - HTML = None + Html = None else: - HTML = WrapDictReplWithEsc(HTML, { + Html = WrapDictReplWithEsc(Html, { #'[staticoso:PageHead]': Meta['Head'], #'': Meta['Head'], # #DEPRECATION # - 'staticoso:Site:Menu': HTMLPagesList, + 'staticoso:Site:Menu': HtmlPagesList, 'staticoso:Page:Lang': Meta['Language'] if Meta['Language'] else f.SiteLang, - 'staticoso:Page:Chapters': HTMLTitles, + 'staticoso:Page:Chapters': HtmlTitles, 'staticoso:Page:Title': Title, 'staticoso:Page:Description': Description, 'staticoso:Page:Image': Image, 'staticoso:Page:Path': PagePath, 'staticoso:Page:Style': Meta['Style'], ################ - 'staticoso:SiteMenu': HTMLPagesList, + 'staticoso:SiteMenu': HtmlPagesList, 'staticoso:PageLang': Meta['Language'] if Meta['Language'] else f.SiteLang, 'staticoso:PageLanguage': Meta['Language'] if Meta['Language'] else f.SiteLang, - 'staticoso:PageSections': HTMLTitles, + 'staticoso:PageSections': HtmlTitles, 'staticoso:PageTitle': Title, 'staticoso:PageDescription': Description, 'staticoso:PageImage': Image, @@ -126,23 +157,24 @@ def PatchHTML(Flags:dict, File, HTML:str, Snippets:dict, HTMLPagesList:str, Page 'staticoso:SiteAbsoluteRoot': f.SiteRoot, 'staticoso:SiteRelativeRoot': RelativeRoot, }, InternalMacrosWraps) + #Html = WhileFuncResultChanges(HandleTransclusions, {"Html": Html, "Caller": File, "Pages": Pages}, 'Html') for e in Meta['Macros']: - HTML = ReplWithEsc(HTML, f"[:{e}:]", Meta['Macros'][e]) + Html = ReplWithEsc(Html, f"[:{e}:]", Meta['Macros'][e]) for e in f.FolderRoots: - HTML = WrapDictReplWithEsc(HTML, { + Html = WrapDictReplWithEsc(Html, { f'staticoso:CustomPath:{e}': f.FolderRoots[e], f'staticoso:Folder:{e}:AbsoluteRoot': f.FolderRoots[e], #DEPRECATED }, InternalMacrosWraps) for e in Categories: - HTML = WrapDictReplWithEsc(HTML, { + Html = WrapDictReplWithEsc(Html, { f'staticoso:Category:{e}': Categories[e], f'staticoso:CategoryList:{e}': Categories[e], }, InternalMacrosWraps) - HTML = ReplWithEsc(HTML, f'[staticoso:Category:{e}]', Categories[e]) #DEPRECATED + Html = ReplWithEsc(Html, f'[staticoso:Category:{e}]', Categories[e]) #DEPRECATED # TODO: Clean this doubling? - ContentHTML = Content - ContentHTML = WrapDictReplWithEsc(ContentHTML, { + ContentHtml = Content + ContentHtml = WrapDictReplWithEsc(ContentHtml, { # #DEPRECATION # '[staticoso:Page:Title]': Title, '[staticoso:Page:Description]': Description, @@ -157,21 +189,22 @@ def PatchHTML(Flags:dict, File, HTML:str, Snippets:dict, HTMLPagesList:str, Page '': f.SiteRoot, '': RelativeRoot, }, InternalMacrosWraps) + #Html = WhileFuncResultChanges(HandleTransclusions, {"Html": Html, "Caller": File, "Pages": Pages}, 'Html') for e in Meta['Macros']: - ContentHTML = ReplWithEsc(ContentHTML, f"[:{e}:]", Meta['Macros'][e]) + ContentHtml = ReplWithEsc(ContentHtml, f"[:{e}:]", Meta['Macros'][e]) for e in f.FolderRoots: - ContentHTML = WrapDictReplWithEsc(ContentHTML, { + ContentHtml = WrapDictReplWithEsc(ContentHtml, { f'staticoso:CustomPath:{e}': f.FolderRoots[e], f'staticoso:Folder:{e}:AbsoluteRoot': f.FolderRoots[e], #DEPRECATED }, InternalMacrosWraps) for e in Categories: - ContentHTML = WrapDictReplWithEsc(ContentHTML, { + ContentHtml = WrapDictReplWithEsc(ContentHtml, { f'staticoso:Category:{e}': Categories[e], f'staticoso:CategoryList:{e}': Categories[e], }, InternalMacrosWraps) - ContentHTML = ReplWithEsc(ContentHTML, f'[staticoso:Category:{e}]', Categories[e]) #DEPRECATED + ContentHtml = ReplWithEsc(ContentHtml, f'[staticoso:Category:{e}]', Categories[e]) #DEPRECATED - return HTML, ContentHTML, Description, Image + return Html, ContentHtml, Description, Image def BuildPagesSearch(Flags:dict, Pages:list, Template:str, Snippets:dict, Locale:dict): SearchContent = '' @@ -187,21 +220,16 @@ def BuildPagesSearch(Flags:dict, Pages:list, Template:str, Snippets:dict, Locale {Page["ContentHtml"]} ''' - return PatchHTML( + return PatchHtml( Flags=Flags, - File='Search.html', - HTML=Template, + Pages=[], + Page={"File": "Search.html", "PagePath": "Search.html", "Content": Base[0] + SearchContent + Base[1], "Titles": [], "Meta": PageMetaDefault}, + Context={"Html": Template, "HtmlPagesList": "", "Categories": []}, Snippets=Snippets, - HTMLPagesList='', - PagePath='Search.html', - Content=Base[0] + SearchContent + Base[1], - Titles=[], - Meta=PageMetaDefault, - Categories=[], Locale=Locale, LightRun=False)[0] -def HandlePage(Flags:dict, Page:list, Pages, Categories, LimitFiles, Snippets:dict, ConfMenu, Locale:dict): +def HandlePage(Flags:dict, Page:list, Pages:list, Categories, LimitFiles, Snippets:dict, ConfMenu, Locale:dict): File, Content, Titles, Meta = Page f = NameSpace(Flags) TemplatesText = Snippets['Templates'] @@ -233,17 +261,12 @@ def HandlePage(Flags:dict, Page:list, Pages, Categories, LimitFiles, Snippets:di For='Menu', MenuStyle=TemplateMeta['MenuStyle']) - HTML, ContentHTML, Description, Image = PatchHTML( + HTML, ContentHTML, Description, Image = PatchHtml( Flags, - File=File, - HTML=TemplatesText[Meta['Template']], + Pages=Pages, + Page={"File": File, "PagePath": PagePath[len(f"{f.OutDir}/"):], "Content": Content, "Titles": Titles, "Meta": Meta}, + Context={"Html": TemplatesText[Meta['Template']], "HtmlPagesList": HTMLPagesList, "Categories": Categories}, Snippets=Snippets, - HTMLPagesList=HTMLPagesList, - PagePath=PagePath[len(f"{f.OutDir}/"):], - Content=Content, - Titles=Titles, - Meta=Meta, - Categories=Categories, Locale=Locale, LightRun=LightRun) @@ -298,17 +321,12 @@ def HandlePage(Flags:dict, Page:list, Pages, Categories, LimitFiles, Snippets:di WriteFile(ContentPagePath, ContentHTML) if not LightRun and 'htmljournal' in ContentHTML.lower(): # Avoid extra cycles - HTML, _, _, _ = PatchHTML( + HTML, _, _, _ = PatchHtml( Flags, - File=File, - HTML=TemplatesText[Meta['Template']], + Pages=Pages, + Page={"File": File, "PagePath": f'{StripExt(File)}.Journal.html', "Content": MakeHTMLJournal(Flags, Locale, f'{StripExt(File)}.html', ContentHTML), "Titles": "", "Meta": Meta}, + Context={"Html": TemplatesText[Meta['Template']], "HtmlPagesList": HTMLPagesList, "Categories": Categories}, Snippets=Snippets, - HTMLPagesList=HTMLPagesList, - PagePath=f'{StripExt(File)}.Journal.html', - Content=MakeHTMLJournal(Flags, Locale, f'{StripExt(File)}.html', ContentHTML), - Titles='', - Meta=Meta, - Categories=Categories, Locale=Locale, LightRun=LightRun) if Flags["JournalRedirect"]: @@ -338,7 +356,7 @@ def ReorderPagesPaths(Paths:dict, Sorting:dict): Paths[Type].reverse() return Paths -def PopulateCategoryLists(Flags:dict, Pages:list, Categories): +def PopulateCategoryLists(Flags:dict, Pages:list, Categories:dict): for Cat in Categories: for Type in ('Page', 'Post'): Categories[Cat] += GetHTMLPagesList( @@ -424,23 +442,4 @@ def MakeSite(Flags:dict, LimitFiles, Snippets, ConfMenu, GlobalMacros:dict, Loca logging.info("Writing Pages") MadePages = WriteProcessedPages(Flags, Pages, Categories, ConfMenu, Snippets, LimitFiles, PoolSize, Locale) - # Do page transclusions here (?) - #while True: - # Operated = False - # for di,Dest in enumerate(MadePages): - # #print(Dest[0]) - # #TempPath = f'{PathPrefix}{Dest["File"]}' - # #LightRun = False if LimitFiles == False or TempPath in LimitFiles else True - # #if not LightRun: - # if '[staticoso:Transclude:' in Dest[4] and (LimitFiles == False or f'{PathPrefix}{Dest[0]}' in LimitFiles): - # for Item in MadePages: - # SrcPrefix = '' if Item[0].startswith('Posts/') else 'Pages/' - # print(SrcPrefix, Item[0]) - # if Item[0] != Dest[0] and f'[staticoso:Transclude:{SrcPrefix}{Item[0]}]' in Dest[4]: - # MadePages[di][4] = ReplWithEsc(Dest[4], f'', Item[4]) - # print(f'[staticoso:Transclude:{SrcPrefix}{Item[0]}]', Item[4]) - # Operated = True - # if not Operated: - # break - return MadePages diff --git a/App/Source/Modules/Utils.py b/App/Source/Modules/Utils.py index c5c7835..9c8b152 100644 --- a/App/Source/Modules/Utils.py +++ b/App/Source/Modules/Utils.py @@ -92,16 +92,24 @@ def FindAllIndex(Str:str, Sub:str): def ReplWithEsc(Str:str, Find:str, Repl:str, Esc:str='\\'): New = '' Sects = Str.split(Find) + # Every time a substring is found for i,e in enumerate(Sects): + # If it's the first split, append it directly to the New string if i == 0: New += e + # Wrapping parts of the escaped substrings in HTML tags is done to avoid multiple calls of this function nullifying escaping elif i > 0: + # If prev. split ends with 2 times the escape (= escaping of the escape) if Sects[i-1].endswith(Esc*2): - New = New[:-1] + Wrap = f'{New[-1]}' + New = New[:-2] + Wrap New += Repl + e + # If prev. split ends with 1 time the escape (escaping of the substring) elif Sects[i-1].endswith(Esc): New = New[:-1] - New += Find + e + Wrap = f'{Find[0]}' + New += Wrap + Find[1:] + e + # If no escape char else: New += Repl + e return New @@ -180,20 +188,30 @@ def PrintProcPercentDots(Proc:dict, DivMult=1): return True return False -def MultiProcFunctWrap(Args:dict): +def MultiProcFuncWrap(Args:dict): PrintProcPercentDots(Args['Process']) - return Args['Process']['Funct'](Args) + return Args['Process']['Func'](Args) -def DoMultiProc(Funct, ArgsCollection:list, Threads:int=cpu_count(), Progress:bool=False): +def DoMultiProc(Func, ArgsCollection:list, Threads:int=cpu_count(), Progress:bool=False): FinalArgsCollection = [] for Index, Args in enumerate(ArgsCollection): FinalArgsCollection.append(Args) - FinalArgsCollection[Index].update({"Process": {"Funct": Funct, "Num": Index, "Count": len(ArgsCollection)}}) + FinalArgsCollection[Index].update({"Process": {"Func": Func, "Num": Index, "Count": len(ArgsCollection)}}) Results = [] if Progress: os.system('printf "["') # Using system print because (see PrintProcPercentDots()) with Pool(Threads) as MultiprocPool: - Results = MultiprocPool.map(MultiProcFunctWrap if Progress else Funct, FinalArgsCollection) + Results = MultiprocPool.map(MultiProcFuncWrap if Progress else Func, FinalArgsCollection) if Progress: os.system('printf "]\n"') # Newline after percentage dots return Results + +# Execute a function, whose output is compared to one input argument, as long as the output is different from the previous cycle; the moment it's equal, return +def WhileFuncResultChanges(Func, Args:dict, ResultKey:str): + Result = Args[ResultKey] + while True: + ResultOld = Result + Args.update({ResultKey: Result}) + Result = Func(**Args) + if ResultOld == Result: + return Result diff --git a/README.md b/README.md index d633612..45f8cc7 100644 --- a/README.md +++ b/README.md @@ -106,3 +106,4 @@ All of this is because some crucial things might be changed from one commit to a - Despite differential building and multithreading, the program still needs some more optimizations. - Ordering pages in the global menu with external configuration flags (outside the pages' source) yields broken and unpredictable results. - If site is small, the graphic of percentage completion is bugged (appears shorter). +- Markdown markup in headings (e.g. `## Title **text**`) is not processed and will render literally (e.g. `

Title **text**

`) diff --git a/Sites/BuildAll.sh b/Sites/BuildAll.sh index 3d4b8cf..7bfcd6c 100644 --- a/Sites/BuildAll.sh +++ b/Sites/BuildAll.sh @@ -4,7 +4,8 @@ cd "$( dirname "$( realpath "$0" )" )" # First build all sites independently, # then move them to a base public folder -mkdir ../public +rm -rf ../public/* +mkdir -p ../public for Site in * do @@ -12,7 +13,10 @@ do then cd $Site python3 ../../App/Source/Build.py - mv ./public ../../public/$Site + cp -r ./public ../../public/$Site cd .. fi done + +cd ../public +mv ./Main/* ./ diff --git a/Sites/Docs/LICENSE.md b/Sites/LICENSE.md similarity index 100% rename from Sites/Docs/LICENSE.md rename to Sites/LICENSE.md diff --git a/Sites/Main/.github/workflows/main.yml b/Sites/Main/.github/workflows/main.yml new file mode 100644 index 0000000..1ecb70b --- /dev/null +++ b/Sites/Main/.github/workflows/main.yml @@ -0,0 +1,33 @@ +name: Build and Deploy with staticoso + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + + workflow_dispatch: + +jobs: + page_build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Build + run: | + sudo apt update + sudo apt install -y software-properties-common + sudo add-apt-repository ppa:deadsnakes/ppa + sudo apt update + sudo apt install -y python3.10 curl git + curl -sS https://bootstrap.pypa.io/get-pip.py | sudo python3.10 + sudo pip3 install lxml + git clone --depth 1 https://gitlab.com/octtspacc/staticoso + python3.10 ./staticoso/Source/Build.py + + - name: Deploy + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: public diff --git a/Sites/Main/.gitlab-ci.yml b/Sites/Main/.gitlab-ci.yml new file mode 100644 index 0000000..5b7d7d3 --- /dev/null +++ b/Sites/Main/.gitlab-ci.yml @@ -0,0 +1,16 @@ +image: alpine:latest + +before_script: | + apk update + apk add python3 git + +pages: + stage: deploy + script: | + git clone --depth 1 https://gitlab.com/octtspacc/staticoso + python3 ./staticoso/Source/Build.py + artifacts: + paths: + - public + rules: + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH diff --git a/Sites/Main/Pages/index.md b/Sites/Main/Pages/index.md new file mode 100644 index 0000000..f2ce3fe --- /dev/null +++ b/Sites/Main/Pages/index.md @@ -0,0 +1,16 @@ +// % Title = staticoso + +## Welcome to the always-WIP world of staticoso... + +Things you can do now: + +
+Cry +: [Read the Docs](./Docs) - you cry here because they're outdated and incomplete +: [Look at the code](https://gitlab.com/octtspacc/staticoso) - you cry here because it's bad +
+ +
+Procrastinate +: [Visit my _sitoctt_](https://sitoctt.octt.eu.org) - involuntarily the flagship staticoso site +
diff --git a/Sites/Main/Site.ini b/Sites/Main/Site.ini new file mode 100644 index 0000000..51324a0 --- /dev/null +++ b/Sites/Main/Site.ini @@ -0,0 +1,2 @@ +[Site] +Name = staticoso diff --git a/Sites/Main/Templates/Default.html b/Sites/Main/Templates/Default.html new file mode 100644 index 0000000..7048248 --- /dev/null +++ b/Sites/Main/Templates/Default.html @@ -0,0 +1,21 @@ + + + + + + + [staticoso:Page:Title] - [staticoso:Site:Name] + + + + + + + + + + [staticoso:Page:Content] + +