From f63b875b76b7befee81f1f786ea92337506a2c91 Mon Sep 17 00:00:00 2001 From: Wolfsblvt Date: Tue, 16 Jul 2024 01:24:03 +0200 Subject: [PATCH] First draft of the macro lexer --- public/script.js | 7 +++ public/scripts/macros/MacroEngine.js | 39 +++++++++++++ public/scripts/macros/MacroLexer.js | 86 ++++++++++++++++++++++++++++ public/scripts/macros/MacroParser.js | 26 +++++++++ 4 files changed, 158 insertions(+) create mode 100644 public/scripts/macros/MacroEngine.js create mode 100644 public/scripts/macros/MacroLexer.js create mode 100644 public/scripts/macros/MacroParser.js diff --git a/public/script.js b/public/script.js index 5a08e7cfb..2043cdaa9 100644 --- a/public/script.js +++ b/public/script.js @@ -242,6 +242,8 @@ import { INTERACTABLE_CONTROL_CLASS, initKeyboard } from './scripts/keyboard.js' import { initDynamicStyles } from './scripts/dynamic-styles.js'; import { SlashCommandEnumValue, enumTypes } from './scripts/slash-commands/SlashCommandEnumValue.js'; import { enumIcons } from './scripts/slash-commands/SlashCommandCommonEnumsProvider.js'; +import { MacroLexer } from './scripts/macros/MacroLexer.js'; +import { MacroEngine } from './scripts/macros/MacroEngine.js'; //exporting functions and vars for mods export { @@ -7910,6 +7912,11 @@ window['SillyTavern'].getContext = function () { Popup: Popup, POPUP_TYPE: POPUP_TYPE, POPUP_RESULT: POPUP_RESULT, + macros: { + MacroLexer, + MacrosParser, + MacroEngine, + }, }; }; diff --git a/public/scripts/macros/MacroEngine.js b/public/scripts/macros/MacroEngine.js new file mode 100644 index 000000000..ecf5a39a9 --- /dev/null +++ b/public/scripts/macros/MacroEngine.js @@ -0,0 +1,39 @@ +import { MacroLexer } from './MacroLexer.js'; +import { MacroParser } from './MacroParser.js'; + +class MacroEngine { + static instance = new MacroEngine(); + + constructor() { + this.parser = MacroParser.instance; + } + + parseDocument(input) { + const lexingResult = MacroLexer.tokenize(input); + this.parser.input = lexingResult.tokens; + // const cst = this.parser.document(); + // return cst; + } + + evaluate(input) { + const lexingResult = MacroLexer.tokenize(input); + this.parser.input = lexingResult.tokens; + // const cst = this.parser.macro(); + + // if (this.parser.errors.length > 0) { + // throw new Error('Parsing errors detected'); + // } + + // return this.execute(cst); + } + + execute(cstNode) { + // Implement execution logic here, traversing the CST and replacing macros with their values + // For now, we'll just return a placeholder result + return 'Executed Macro'; + } +} + +const macroEngineInstance = MacroEngine.instance; + +export { MacroEngine, macroEngineInstance }; diff --git a/public/scripts/macros/MacroLexer.js b/public/scripts/macros/MacroLexer.js new file mode 100644 index 000000000..d54c3e4dc --- /dev/null +++ b/public/scripts/macros/MacroLexer.js @@ -0,0 +1,86 @@ +import { createToken, Lexer } from '../../lib/chevrotain.min.mjs'; + +/** @enum {string} */ +const MODES = { + macro: 'macro_mode', + text: 'text_mode', +}; + +/** @readonly */ +const tokens = { + // General capture-all plaintext without macros + Plaintext: createToken({ name: 'Plaintext', pattern: /(.+?)(?=\{\{)|(.+)/, line_breaks: true }), // Match everything up till opening brackets. Or to the end. + + // The relevant blocks to start/end a macro + MacroStart: createToken({ name: 'MacroStart', pattern: /\{\{/, push_mode: MODES.macro }), + MacroEnd: createToken({ name: 'MacroEnd', pattern: /\}\}/, pop_mode: true }), + + // All tokens that can be captured inside a macro + DoubleColon: createToken({ name: 'DoubleColon', pattern: /::/ }), + Colon: createToken({ name: 'Colon', pattern: /:/ }), + Equals: createToken({ name: 'Equals', pattern: /=/ }), + Quote: createToken({ name: 'Quote', pattern: /"/ }), + Identifier: createToken({ name: 'Identifier', pattern: /[a-zA-Z_]\w*/ }), + WhiteSpace: createToken({ + name: 'WhiteSpace', + pattern: /\s+/, + group: Lexer.SKIPPED, + }), + // TODO: Capture-all rest for now, that is not the macro end or opening of a new macro. Might be replaced later down the line. + Text: createToken({ name: 'Text', pattern: /.+(?=\}\}|\{\{)/, line_breaks: true }), +}; + +/** + * The singleton instance of the MacroLexer. + * + * @type {MacroLexer} + */ +let instance; +export { instance as MacroLexer }; + +class MacroLexer extends Lexer { + /** @type {MacroLexer} */ static #instance; + /** @type {MacroLexer} */ static get instance() { return MacroLexer.#instance ?? (MacroLexer.#instance = new MacroLexer()); } + + // Define the tokens + /** @readonly */ static tokens = tokens; + /** @readonly */ static def = { + modes: { + [MODES.text]: [ + tokens.MacroStart, + tokens.Plaintext, + ], + [MODES.macro]: [ + tokens.MacroStart, + tokens.MacroEnd, + tokens.DoubleColon, + tokens.Colon, + tokens.Equals, + tokens.Quote, + tokens.Identifier, + tokens.WhiteSpace, + tokens.Text, + ], + }, + defaultMode: MODES.text, + }; + /** @readonly */ tokens = tokens; + /** @readonly */ def = MacroLexer.def; + + /** @private */ + constructor() { + super(MacroLexer.def); + } + + test(input) { + const result = this.tokenize(input); + return { + errors: result.errors, + groups: result.groups, + tokens: result.tokens.map(({ tokenType, ...rest }) => ({ type: tokenType.name, ...rest, tokenType: tokenType })), + }; + } +} + +instance = MacroLexer.instance; + diff --git a/public/scripts/macros/MacroParser.js b/public/scripts/macros/MacroParser.js new file mode 100644 index 000000000..9c80f8623 --- /dev/null +++ b/public/scripts/macros/MacroParser.js @@ -0,0 +1,26 @@ +import { CstParser } from './lib.js'; +import { MacroLexer } from './MacroLexer.js'; + +/** + * The singleton instance of the MacroParser. + * + * @type {MacroParser} + */ +let instance; +export { instance as MacroParser }; + +class MacroParser extends CstParser { + /** @type {MacroParser} */ static #instance; + /** @type {MacroParser} */ static get instance() { return MacroParser.#instance ?? (MacroParser.#instance = new MacroParser()); } + + /** @private */ + constructor() { + super(MacroLexer.def); + + // const $ = this; + + this.performSelfAnalysis(); + } +} + +instance = MacroParser.instance;