basics for new parser

public/scripts/slash-commands/SlashCommand.js

@@ -0,0 +1,9 @@
+export class SlashCommand {
+    /**@type {String}*/ name;
+    /**@type {Function}*/ callback;
+    /**@type {String}*/ helpString;
+    /**@type {String}*/ helpStringFormatted;
+    /**@type {Boolean}*/ interruptsGeneration;
+    /**@type {Boolean}*/ purgeFromMessage;
+    /**@type {String[]}*/ aliases;
+}

public/scripts/slash-commands/SlashCommandClosure.js

@@ -0,0 +1,123 @@
+import { substituteParams } from '../../script.js';
+import { SlashCommandClosureResult } from './SlashCommandClosureResult.js';
+import { SlashCommandExecutor } from './SlashCommandExecutor.js';
+import { SlashCommandScope } from './SlashCommandScope.js';
+
+export class SlashCommandClosure {
+    /**@type {SlashCommandScope}*/ scope;
+    /**@type {Boolean}*/ executeNow = false;
+    /**@type {SlashCommandExecutor[]}*/ executorList = [];
+    /**@type {String}*/ keptText;
+
+    constructor(parent) {
+        this.scope = new SlashCommandScope(parent);
+    }
+
+    substituteParams(text) {
+        text = substituteParams(text)
+            .replace(/{{pipe}}/g, this.scope.pipe)
+            .replace(/{{var::(\w+?)}}/g, (_, key)=>this.scope.getVariable(key))
+        ;
+        for (const key of Object.keys(this.scope.macros)) {
+            text = text.replace(new RegExp(`{{${key}}}`), this.scope.macros[key]);
+        }
+        return text;
+    }
+
+    getCopy() {
+        const closure = new SlashCommandClosure();
+        closure.scope = this.scope.getCopy();
+        closure.executeNow = this.executeNow;
+        closure.executorList = this.executorList;
+        closure.keptText = this.keptText;
+        return closure;
+    }
+
+    async execute() {
+        const closure = this.getCopy();
+        return await closure.executeDirect();
+    }
+
+    async executeDirect() {
+        let interrupt = false;
+
+        for (const executor of this.executorList) {
+            interrupt = executor.command.interruptsGeneration;
+            let args = {
+                _scope: this.scope,
+            };
+            let value;
+            // substitute named arguments
+            for (const key of Object.keys(executor.args)) {
+                if (executor.args[key] instanceof SlashCommandClosure) {
+                    /**@type {SlashCommandClosure}*/
+                    const closure = executor.args[key];
+                    closure.scope.parent = this.scope;
+                    if (closure.executeNow) {
+                        args[key] = (await closure.execute())?.pipe;
+                    } else {
+                        args[key] = closure;
+                    }
+                } else {
+                    args[key] = this.substituteParams(executor.args[key]);
+                }
+                // unescape named argument
+                if (typeof args[key] == 'string') {
+                    args[key] = args[key]
+                        ?.replace(/\\\|/g, '|')
+                        ?.replace(/\\\{/g, '{')
+                        ?.replace(/\\\}/g, '}')
+                    ;
+                }
+            }
+
+            // substitute unnamed argument
+            if (executor.value === undefined) {
+                value = this.scope.pipe;
+            } else if (executor.value instanceof SlashCommandClosure) {
+                /**@type {SlashCommandClosure}*/
+                const closure = executor.value;
+                closure.scope.parent = this.scope;
+                if (closure.executeNow) {
+                    value = (await closure.execute())?.pipe;
+                } else {
+                    value = closure;
+                }
+            } else if (Array.isArray(executor.value)) {
+                value = [];
+                for (let i = 0; i < executor.value.length; i++) {
+                    let v = executor.value[i];
+                    if (v instanceof SlashCommandClosure) {
+                        /**@type {SlashCommandClosure}*/
+                        const closure = v;
+                        closure.scope.parent = this.scope;
+                        if (closure.executeNow) {
+                            v = (await closure.execute())?.pipe;
+                        } else {
+                            v = closure;
+                        }
+                    } else {
+                        v = this.substituteParams(v);
+                    }
+                    value[i] = v;
+                }
+                if (!value.find(it=>it instanceof SlashCommandClosure)) {
+                    value = value.join(' ');
+                }
+            } else {
+                value = this.substituteParams(executor.value);
+            }
+            // unescape unnamed argument
+            if (typeof value == 'string') {
+                value = value
+                    ?.replace(/\\\|/g, '|')
+                    ?.replace(/\\\{/g, '{')
+                    ?.replace(/\\\}/g, '}')
+                ;
+            }
+
+            this.scope.pipe = await executor.command.callback(args, value);
+        }
+        return Object.assign(new SlashCommandClosureResult(), { interrupt, newText: this.keptText, pipe: this.scope.pipe });
+    }
+}

public/scripts/slash-commands/SlashCommandClosureResult.js

@@ -0,0 +1,5 @@
+export class SlashCommandClosureResult {
+    /**@type {Boolean}*/ interrupt = false;
+    /**@type {String}*/ newText = '';
+    /**@type {String}*/ pipe;
+}

public/scripts/slash-commands/SlashCommandExecutor.js

@@ -0,0 +1,18 @@
+// eslint-disable-next-line no-unused-vars
+import { SlashCommand } from './SlashCommand.js';
+// eslint-disable-next-line no-unused-vars
+import { SlashCommandClosure } from './SlashCommandClosure.js';
+
+export class SlashCommandExecutor {
+    /**@type {Number}*/ start;
+    /**@type {Number}*/ end;
+    /**@type {String}*/ name = '';
+    /**@type {SlashCommand}*/ command;
+    // @ts-ignore
+    /**@type {Map<String,String|SlashCommandClosure>}*/ args = {};
+    /**@type {String|SlashCommandClosure|(String|SlashCommandClosure)[]}*/ value;
+
+    constructor(start) {
+        this.start = start;
+    }
+}

public/scripts/slash-commands/SlashCommandParser.js

@@ -0,0 +1,266 @@
+import { SlashCommand } from './SlashCommand.js';
+import { SlashCommandClosure } from './SlashCommandClosure.js';
+import { SlashCommandExecutor } from './SlashCommandExecutor.js';
+import { SlashCommandParserError } from './SlashCommandParserError.js';
+// eslint-disable-next-line no-unused-vars
+import { SlashCommandScope } from './SlashCommandScope.js';
+
+export class SlashCommandParser {
+    // @ts-ignore
+    /**@type {Map<String, SlashCommand>}*/ commands = {};
+    // @ts-ignore
+    /**@type {Map<String, String>}*/ helpStrings = {};
+    /**@type {String}*/ text;
+    /**@type {String}*/ keptText;
+    /**@type {Number}*/ index;
+    /**@type {SlashCommandScope}*/ scope;
+
+    /**@type {SlashCommandExecutor[]}*/ commandIndex;
+
+    get ahead() {
+        return this.text.slice(this.index + 1);
+    }
+    get behind() {
+        return this.text.slice(0, this.index);
+    }
+    get char() {
+        return this.text[this.index];
+    }
+    get endOfText() {
+        return this.index >= this.text.length || /^\s+$/.test(this.ahead);
+    }
+
+    addCommand(command, callback, aliases, helpString = '', interruptsGeneration = false, purgeFromMessage = true) {
+        if (['/', '#'].includes(command[0])) {
+            throw new Error(`Illegal Name. Slash commandn name cannot begin with "${command[0]}".`);
+        }
+        const fnObj = Object.assign(new SlashCommand(), { name:command, callback, helpString, interruptsGeneration, purgeFromMessage, aliases });
+
+        if ([command, ...aliases].some(x => Object.hasOwn(this.commands, x))) {
+            console.trace('WARN: Duplicate slash command registered!');
+        }
+
+        this.commands[command] = fnObj;
+
+        if (Array.isArray(aliases)) {
+            aliases.forEach((alias) => {
+                this.commands[alias] = fnObj;
+            });
+        }
+
+        let stringBuilder = `<span class="monospace">/${command}</span> ${helpString} `;
+        if (Array.isArray(aliases) && aliases.length) {
+            let aliasesString = `(alias: ${aliases.map(x => `<span class="monospace">/${x}</span>`).join(', ')})`;
+            stringBuilder += aliasesString;
+        }
+        this.helpStrings[command] = stringBuilder;
+        fnObj.helpStringFormatted = stringBuilder;
+    }
+
+    getHelpString() {
+        const listItems = Object
+            .entries(this.helpStrings)
+            .sort((a, b) => a[0].localeCompare(b[0]))
+            .map(x => x[1])
+            .map(x => `<li>${x}</li>`)
+            .join('\n');
+        return `<p>Slash commands:</p><ol>${listItems}</ol>
+        <small>Slash commands can be batched into a single input by adding a pipe character | at the end, and then writing a new slash command.</small>
+        <ul><li><small>Example:</small><code>/cut 1 | /sys Hello, | /continue</code></li>
+        <li>This will remove the first message in chat, send a system message that starts with 'Hello,', and then ask the AI to continue the message.</li></ul>`;
+    }
+
+    getCommandAt(text, index) {
+        try {
+            this.parse(text);
+        } catch (e) {
+            // do nothing
+        }
+        index += 2;
+        return this.commandIndex.filter(it=>it.start <= index && (it.end >= index || it.end == null)).slice(-1)[0]
+            ?? null
+        ;
+    }
+
+    take(length = 1, keep = false) {
+        let content = '';
+        while (length-- > 0) {
+            content += this.char;
+            this.index++;
+        }
+        if (keep) this.keptText += content;
+        return content;
+    }
+
+
+    parse(text) {
+        this.text = `{:${text}:}`;
+        this.keptText = '';
+        this.index = 0;
+        this.scope = null;
+        this.commandIndex = [];
+        const closure = this.parseClosure();
+        console.log('[STS]', closure);
+        closure.keptText = this.keptText;
+        return closure;
+    }
+
+    testClosure() {
+        return this.ahead.length > 0 && this.char == '{' && this.ahead[0] == ':';
+    }
+    testClosureEnd() {
+        if (this.ahead.length < 1) throw new SlashCommandParserError(`Unclosed closure at position ${this.index - 2}`, this.text, this.index);
+        return this.char == ':' && this.ahead[0] == '}' && this.behind.slice(-1) != '\\';
+    }
+    parseClosure() {
+        this.take(2); // discard opening {:
+        let closure = new SlashCommandClosure(this.scope);
+        this.scope = closure.scope;
+        while (/\s/.test(this.char)) this.take(); // discard whitespace
+        while (!this.testClosureEnd()) {
+            if (this.testCommand()) {
+                closure.executorList.push(this.parseCommand());
+            } else {
+                while (!this.testCommandEnd()) this.take(1); // discard plain text and comments
+            }
+            while (/\s|\|/.test(this.char)) this.take(); // discard whitespace and pipe (command separator)
+        }
+        this.take(2); // discard closing :}
+        if (this.char == '(' && this.ahead[0] == ')') {
+            this.take(2); // discard ()
+            closure.executeNow = true;
+        }
+        while (/\s/.test(this.char)) this.take(); // discard trailing whitespace
+        this.scope = closure.scope.parent;
+        return closure;
+    }
+
+    testCommand() {
+        return this.char == '/' && this.behind.slice(-1) != '\\' && !['/', '#'].includes(this.ahead[0]);
+    }
+    testCommandEnd() {
+        return this.testClosureEnd() || this.endOfText || (this.char == '|' && this.behind.slice(-1) != '\\');
+    }
+    parseCommand() {
+        const start = this.index;
+        const cmd = new SlashCommandExecutor(start);
+        this.take(); // discard "/"
+        this.commandIndex.push(cmd);
+        while (!/\s/.test(this.char) && !this.testCommandEnd()) cmd.name += this.take(); // take chars until whitespace or end
+        while (/\s/.test(this.char)) this.take(); // discard whitespace
+        if (!this.commands[cmd.name]) throw new SlashCommandParserError(`Unknown command at position ${this.index - cmd.name.length - 2}: "/${cmd.name}"`, this.text, this.index - cmd.name.length);
+        cmd.command = this.commands[cmd.name];
+        while (this.testNamedArgument()) {
+            const arg = this.parseNamedArgument();
+            cmd.args[arg.key] = arg.value;
+            while (/\s/.test(this.char)) this.take(); // discard whitespace
+        }
+        while (/\s/.test(this.char)) this.take(); // discard whitespace
+        if (this.testUnnamedArgument()) {
+            cmd.value = this.parseUnnamedArgument();
+        }
+        if (this.testCommandEnd()) {
+            cmd.end = this.index;
+            if (!cmd.command.purgeFromMessage) this.keptText += this.text.slice(cmd.start, cmd.end);
+            return cmd;
+        } else {
+            console.warn(this.behind, this.char, this.ahead);
+            throw new SlashCommandParserError(`Unexpected end of command at position ${this.index - 2}: "/${cmd.command}"`, this.text, this.index);
+        }
+    }
+
+    testNamedArgument() {
+        return /^(\w+)=/.test(`${this.char}${this.ahead}`);
+    }
+    parseNamedArgument() {
+        let key = '';
+        while (/\w/.test(this.char)) key += this.take(); // take chars
+        this.take(); // discard "="
+        let value;
+        if (this.testClosure()) {
+            value = this.parseClosure();
+        } else if (this.testQuotedValue()) {
+            value = this.parseQuotedValue();
+        } else if (this.testListValue()) {
+            value = this.parseListValue();
+        } else if (this.testValue()) {
+            value = this.parseValue();
+        }
+        return { key, value };
+    }
+
+    testUnnamedArgument() {
+        return !this.testCommandEnd();
+    }
+    testUnnamedArgumentEnd() {
+        return this.testCommandEnd();
+    }
+    parseUnnamedArgument() {
+        /**@type {SlashCommandClosure|String}*/
+        let value = '';
+        let isList = false;
+        let listValues = [];
+        while (!this.testUnnamedArgumentEnd()) {
+            if (this.testClosure()) {
+                isList = true;
+                if (value.length > 0) {
+                    listValues.push(value.trim());
+                    value = '';
+                }
+                listValues.push(this.parseClosure());
+            } else {
+                value += this.take();
+            }
+        }
+        if (isList && value.trim().length > 0) {
+            listValues.push(value.trim());
+        }
+        if (isList) {
+            if (listValues.length == 1) return listValues[0];
+            return listValues;
+        }
+        return value.trim();
+    }
+
+    testQuotedValue() {
+        return this.char == '"' && this.behind.slice(-1) != '\\';
+    }
+    testQuotedValueEnd() {
+        if (this.endOfText) throw new SlashCommandParserError(`Unexpected end of quoted value at position ${this.index}`, this.text, this.index);
+        return this.char == '"' && this.behind.slice(-1) != '\\';
+    }
+    parseQuotedValue() {
+        this.take(); // discard opening quote
+        let value = '';
+        while (!this.testQuotedValueEnd()) value += this.take(); // take all chars until closing quote
+        this.take(); // discard closing quote
+        return value;
+    }
+
+    testListValue() {
+        return this.char == '[' && this.behind.slice(-1) != '\\';
+    }
+    testListValueEnd() {
+        if (this.endOfText) throw new SlashCommandParserError(`Unexpected end of list value at position ${this.index}`, this.text, this.index);
+        return this.char == ']' && this.behind.slice(-1) != '\\';
+    }
+    parseListValue() {
+        let value = '';
+        while (!this.testListValueEnd()) value += this.take(); // take all chars until closing bracket
+        value += this.take(); // take closing bracket
+        return value;
+    }
+
+    testValue() {
+        return !/\s/.test(this.char);
+    }
+    testValueEnd() {
+        if (/\s/.test(this.char)) return true;
+        return this.testCommandEnd();
+    }
+    parseValue() {
+        let value = '';
+        while (!this.testValueEnd()) value += this.take(); // take all chars until value end
+        return value;
+    }
+}

public/scripts/slash-commands/SlashCommandParserError.js

@@ -0,0 +1,47 @@
+export class SlashCommandParserError extends Error {
+    /**@type {String}*/ text;
+    /**@type {Number}*/ index;
+
+    get line() {
+        return this.text.slice(0, this.index).replace(/[^\n]/g, '').length;
+    }
+    get column() {
+        return this.text.slice(0, this.index).split('\n').pop().length;
+    }
+    get hint() {
+        let lineOffset = this.line.toString().length;
+        let lineStart = this.index;
+        let start = this.index;
+        let end = this.index;
+        let offset = 0;
+        let lineCount = 0;
+        while (offset < 10000 && lineCount < 3 && start >= 0) {
+            if (this.text[start] == '\n') lineCount++;
+            if (lineCount == 0) lineStart--;
+            offset++;
+            start--;
+        }
+        if (this.text[start + 1] == '\n') start++;
+        offset = 0;
+        while (offset < 10000 && this.text[end] != '\n') {
+            offset++;
+            end++;
+        }
+        let hint  = [];
+        let lines = this.text.slice(start + 1, end - 1).split('\n');
+        let lineNum = this.line - lines.length + 1;
+        for (const line of lines) {
+            const num = `${' '.repeat(lineOffset - lineNum.toString().length)}${lineNum}`;
+            lineNum++;
+            hint.push(`${num}:  ${line}`);
+        }
+        hint.push(`${' '.repeat(this.index - lineStart + lineOffset + 1)}^^^^^`);
+        return hint.join('\n');
+    }
+
+    constructor(message, text, index) {
+        super(message);
+        this.text = text.slice(2, -2);
+        this.index = index - 2;
+    }
+}

public/scripts/slash-commands/SlashCommandScope.js

@@ -0,0 +1,102 @@
+export class SlashCommandScope {
+    // @ts-ignore
+    /**@type {Map<String, Object>}*/ variables = {};
+    // @ts-ignore
+    /**@type {Map<String, Object>}*/ macros = {};
+    /**@type {SlashCommandScope}*/ parent;
+    /**@type {String}*/ #pipe;
+    get pipe() {
+        return this.#pipe ?? this.parent?.pipe;
+    }
+    set pipe(value) {
+        this.#pipe = value;
+    }
+
+
+    constructor(parent) {
+        this.parent = parent;
+    }
+
+    getCopy() {
+        const scope = new SlashCommandScope(this.parent);
+        scope.variables = Object.assign({}, this.variables);
+        scope.macros = this.macros;
+        scope.#pipe = this.#pipe;
+        return scope;
+    }
+
+
+    setMacro(key, value) {
+        this.macros[key] = value;
+    }
+
+
+    existsVariableInScope(key) {
+        return Object.keys(this.variables).includes(key);
+    }
+    existsVariable(key) {
+        return Object.keys(this.variables).includes(key) || this.parent?.existsVariable(key);
+    }
+    letVariable(key, value = undefined) {
+        if (this.existsVariableInScope(key)) throw new SlashCommandScopeVariableExistsError(`Variable named "${key}" already exists.`);
+        this.variables[key] = value;
+    }
+    setVariable(key, value, index = null) {
+        if (this.existsVariableInScope(key)) {
+            if (index !== null && index !== undefined) {
+                let v = this.variables[key];
+                try {
+                    v = JSON.parse(v);
+                    const numIndex = Number(index);
+                    if (Number.isNaN(numIndex)) {
+                        v[index] = value;
+                    } else {
+                        v[numIndex] = value;
+                    }
+                    v = JSON.stringify(v);
+                } catch {
+                    v[index] = value;
+                }
+                this.variables[key] = v;
+            } else {
+                this.variables[key] = value;
+            }
+            return value;
+        }
+        if (this.parent) {
+            return this.parent.setVariable(key, value, index);
+        }
+        throw new SlashCommandScopeVariableNotFoundError(`No such variable: "${key}"`);
+    }
+    getVariable(key, index = null) {
+        if (this.existsVariableInScope(key)) {
+            if (index !== null && index !== undefined) {
+                let v = this.variables[key];
+                try { v = JSON.parse(v); } catch { /* empty */ }
+                const numIndex = Number(index);
+                if (Number.isNaN(numIndex)) {
+                    v = v[index];
+                } else {
+                    v = v[numIndex];
+                }
+                if (typeof v == 'object') return JSON.stringify(v);
+                return v;
+            } else {
+                const value = this.variables[key];
+                return (value === '' || isNaN(Number(value))) ? (value || '') : Number(value);
+            }
+        }
+        if (this.parent) {
+            return this.parent.getVariable(key, index);
+        }
+        throw new SlashCommandScopeVariableNotFoundError(`No such variable: "${key}"`);
+    }
+}
+
+
+
+
+export class SlashCommandScopeVariableExistsError extends Error {}
+
+
+export class SlashCommandScopeVariableNotFoundError extends Error {}