Extensions: Add regex engine

Regex is a method that is commonly used to find and replace parts
of a string using a single pattern. Add support for using regex in
SillyTavern which allows users to dynamically change various aspects
of the chatting experience.

Users are able to choose where a given regex script should apply
(both invasive and non-invasive options!). Invasive options alter
chat history while non-invasive alters markdown display for the
entire chat.

A new variable called {{match}} is added in regex scripts which
substitutes in the found match from the original find regex script.

There is a lot more that can be added to this extension, but for now,
this is enough.

Signed-off-by: kingbri <bdashore3@proton.me>

public/scripts/extensions/regex/dropdown.html

@@ -0,0 +1,17 @@
+<div class="regex_settings">
+    <div class="inline-drawer">
+        <div class="inline-drawer-toggle inline-drawer-header">
+            <b>Regex</b>
+            <div class="inline-drawer-icon fa-solid fa-circle-chevron-down down"></div>
+        </div>
+        <div class="inline-drawer-content">
+            <div id="open_regex_editor" class="menu_button">
+                <i class="fa-solid fa-pen-to-square"></i>
+                <span>Open Editor</span>
+            </div>
+            <hr />
+            <label>Saved Scripts</label>
+            <div id="saved_regex_scripts" class="flex-container regex-script-container flexFlowColumn"></div>
+        </div>
+    </div>
+</div>

public/scripts/extensions/regex/editor.html

@@ -0,0 +1,84 @@
+<div id="regex_editor_template">
+    <div class="regex_editor">
+        <h3><strong>Regex Editor</strong></h3>
+
+        <div class="flex-container flexFlowColumn">
+            <div class="flex1">
+                <label for="regex_script_name" class="title_restorable">
+                    <small data-i18n="Script Name">Script Name</small>
+                </label>
+                <div>
+                    <input class="regex_script_name text_pole textarea_compact" type="text" maxlength="100" />
+                </div>
+            </div>
+            <div class="flex1">
+                <label for="find_regex" class="title_restorable">
+                    <small data-i18n="Find Regex">Find Regex</small>
+                </label>
+                <div>
+                    <input class="find_regex text_pole textarea_compact" type="text" maxlength="100" />
+                </div>
+            </div>
+            <div class="flex1">
+                <label for="replace_string" class="title_restorable">
+                    <small data-i18n="Replace With">Replace With</small>
+                </label>
+                <div>
+                    <textarea 
+                        class="regex_replace_string text_pole wide100p textarea_compact"
+                        placeholder="Use {{match}} to include the matched text from Find Regex"
+                        rows="2"
+                        maxlength="100"
+                    ></textarea>
+                </div>
+            </div>
+        </div>
+
+        <div class="flex-container">
+            <div class="wi-enter-footer-text flex-container flexFlowColumn flexNoGap align-start">
+                <small>Placement</small>
+                <div>
+                    <label class="checkbox flex-container">
+                        <input type="checkbox" name="replace_position" value="0">
+                        <span data-i18n="Author's Note">Markdown Display</span>
+                    </label>
+                </div>
+                <div>
+                    <label class="checkbox flex-container">
+                        <input type="checkbox" name="replace_position" value="1">
+                        <span data-i18n="Before Char">User Input</span>
+                    </label>
+                </div>
+                <div>
+                    <label class="checkbox flex-container">
+                        <input type="checkbox" name="replace_position" value="2">
+                        <span data-i18n="After Char">AI Output</span>
+                    </label>
+                </div>
+                <div>
+                    <label class="checkbox flex-container">
+                        <input type="checkbox" name="replace_position" value="3">
+                        <span data-i18n="Author's Note">/sys Command</span>
+                    </label>
+                </div>
+                <div>
+                    <label class="checkbox flex-container">
+                        <input type="checkbox" name="replace_position" value="4">
+                        <span data-i18n="Author's Note">/sendas Command</span>
+                    </label>
+                </div>
+            </div>
+            <div class="wi-enter-footer-text flex-container flexFlowColumn flexNoGap align-start">
+                <small>Other Options</small>
+                <label class="checkbox flex-container">
+                    <input type="checkbox" name="disabled" />
+                    <span data-i18n="Disabled">Disabled</span>
+                </label>
+                <label class="checkbox flex-container">
+                    <input type="checkbox" name="run_on_edit" />
+                    <span data-i18n="Run On Edit">Run On Edit</span>
+                </label>
+            </div>
+        </div>
+    </div>
+</div>

public/scripts/extensions/regex/engine.js

@@ -0,0 +1,58 @@
+import { substituteParams } from "../../../script.js";
+export {
+    runRegexScript
+}
+
+// From: https://github.com/IonicaBizau/regex-parser.js/blob/master/lib/index.js
+function regexFromString(input) {
+    // Parse input
+    var m = input.match(/(\/?)(.+)\1([a-z]*)/i);
+
+    // Invalid flags
+    if (m[3] && !/^(?!.*?(.).*?\1)[gmixXsuUAJ]+$/.test(m[3])) {
+        return RegExp(input);
+    }
+
+    // Create the regular expression
+    return new RegExp(m[2], m[3]);
+}
+
+// Runs the provided regex script on the given string
+function runRegexScript(regexScript, rawString) {
+    if (!!(regexScript.disabled)) {
+        return;
+    }
+
+    let match;
+    let newString;
+    const findRegex = regexFromString(regexScript.findRegex);
+    while ((match = findRegex.exec(rawString)) !== null) {
+        const fencedMatch = match[0];
+        const capturedMatch = match[1];
+
+        // TODO: Use substrings for replacement. But not necessary at this time.
+        // A substring is from match.index to match.index + match[0].length or fencedMatch.length
+        const subReplaceString = substituteRegexParams(regexScript.replaceString, { regexMatch: capturedMatch ?? fencedMatch });
+        if (!newString) {
+            newString = rawString.replace(fencedMatch, subReplaceString);
+        } else {
+            newString = newString.replace(fencedMatch, subReplaceString);
+        }
+
+        // If the regex isn't global, break out of the loop
+        if (!findRegex.flags.includes('g')) {
+            break;
+        }
+    }
+
+    return newString;
+}
+
+// Substitutes parameters 
+function substituteRegexParams(rawString, { regexMatch }) {
+    let finalString = rawString;
+    finalString = finalString.replace("{{match}}", regexMatch);
+    finalString = substituteParams(finalString);
+
+    return finalString;
+}

public/scripts/extensions/regex/index.js

@@ -0,0 +1,155 @@
+import { callPopup, eventSource, event_types, saveSettingsDebounced } from "../../../script.js";
+import { extension_settings } from "../../extensions.js";
+import { uuidv4 } from "../../utils.js";
+export { REGEX_PLACEMENT }
+
+const REGEX_PLACEMENT = {
+    mdDisplay: 0,
+    userInput: 1,
+    aiOutput: 2,
+    system: 3,
+    sendas: 4
+}
+
+async function saveRegexScript(regexScript, existingScriptIndex) {
+    // If the script index is undefined or already exists
+    // Don't fire this if editing an existing script
+    if (existingScriptIndex === -1) {
+        if (!regexScript.scriptName) {
+            toastr.error(`Could not save regex: The script name was undefined or empty!`);
+            return;
+        }
+    
+        if (extension_settings.regex.find((e) => e.scriptName === regexScript.scriptName)) {
+            toastr.error(`Could not save regex: The name ${regexScript.scriptName} already exists.`);
+            return;
+        }
+    }
+
+    if (regexScript.placement.length === 0) {
+        toastr.error(`Could not save regex: One placement checkbox must be selected!`);
+        return;
+    }
+
+    if (existingScriptIndex !== -1) {
+        extension_settings.regex[existingScriptIndex] = regexScript;
+    } else {
+        extension_settings.regex.push(regexScript);
+    }
+
+    saveSettingsDebounced();
+    await loadRegexScripts();
+}
+
+async function deleteRegexScript({ existingId }) {
+    let scriptName = $(`#${existingId}`).find('.regex_script_name').text();
+
+    const existingScriptIndex = extension_settings.regex.findIndex((script) => script.scriptName === scriptName);
+    if (!existingScriptIndex || existingScriptIndex !== -1) {
+        extension_settings.regex.splice(existingScriptIndex, 1);
+
+        saveSettingsDebounced();
+        await loadRegexScripts();
+    }
+}
+
+async function loadRegexScripts() {
+    $("#saved_regex_scripts").empty();
+
+    const scriptTemplate = $(await $.get("scripts/extensions/regex/scriptTemplate.html"));
+
+    extension_settings.regex.forEach((script) => {
+        // Have to clone here
+        const scriptHtml = scriptTemplate.clone();
+        scriptHtml.attr('id', uuidv4());
+        scriptHtml.find('.regex_script_name').text(script.scriptName);
+        scriptHtml.find('.edit_existing_regex').on('click', async function() {
+            await onRegexEditorOpenClick(scriptHtml.attr("id"));
+        });
+        scriptHtml.find('.delete_regex').on('click', async function() {
+            await deleteRegexScript({ existingId: scriptHtml.attr("id") });
+        });
+
+        $("#saved_regex_scripts").append(scriptHtml);
+    });
+}
+
+async function onRegexEditorOpenClick(existingId) {
+    const editorHtml = $(await $.get("scripts/extensions/regex/editor.html"));
+
+    // If an ID exists, fill in all the values
+    let existingScriptIndex = -1;
+    if (existingId) {
+        const existingScriptName = $(`#${existingId}`).find('.regex_script_name').text();
+        existingScriptIndex = extension_settings.regex.findIndex((script) => script.scriptName === existingScriptName);
+        if (existingScriptIndex !== -1) {
+            const existingScript = extension_settings.regex[existingScriptIndex];
+            editorHtml.find(`.regex_script_name`).val(existingScript.scriptName);
+            editorHtml.find(`.find_regex`).val(existingScript.findRegex);
+            editorHtml.find(`.regex_replace_string`).val(existingScript.replaceString);
+            editorHtml
+                .find(`input[name="disabled"]`)
+                .prop("checked", existingScript.disabled);
+            editorHtml
+                .find(`input[name="run_on_edit"]`)
+                .prop("checked", existingScript.runOnEdit);
+
+            existingScript.placement.forEach((element) => {
+                editorHtml
+                    .find(`input[name="replace_position"][value="${element}"]`)
+                    .prop("checked", true);
+            });
+        }
+    } else {
+        editorHtml
+            .find(`input[name="run_on_edit"]`)
+            .prop("checked", true);
+
+        editorHtml
+            .find(`input[name="replace_position"][value="0"]`)
+            .prop("checked", true);
+    }
+
+    const popupResult = await callPopup(editorHtml, "confirm", undefined, "Save");
+    if (popupResult) {
+        const newRegexScript = {
+            scriptName: editorHtml.find(".regex_script_name").val(),
+            findRegex: editorHtml.find(".find_regex").val(),
+            replaceString: editorHtml.find(".regex_replace_string").val(),
+            placement:
+                editorHtml
+                    .find(`input[name="replace_position"]`)
+                    .filter(":checked")
+                    .map(function() { return parseInt($(this).val()) })
+                    .get()
+                    .filter((e) => e !== NaN) ?? [],
+            disabled:
+                editorHtml
+                    .find(`input[name="disabled"]`)
+                    .prop("checked"),
+            runOnEdit:
+                editorHtml
+                    .find(`input[name="run_on_edit"]`)
+                    .prop("checked")
+        };
+
+        saveRegexScript(newRegexScript, existingScriptIndex);
+    }
+}
+
+function hookToEvents() {
+    eventSource.on(event_types.SETTINGS_LOADED, async function () {
+        await loadRegexScripts();
+    });
+}
+
+jQuery(async () => {
+    const settingsHtml = await $.get("scripts/extensions/regex/dropdown.html");
+    $("#extensions_settings2").append(settingsHtml);
+    $("#open_regex_editor").on("click", function() {
+        onRegexEditorOpenClick(false);
+    });
+
+    // Listen to event source after 1ms
+    setTimeout(() => hookToEvents(), 1)
+});

public/scripts/extensions/regex/manifest.json

@@ -0,0 +1,11 @@
+{
+    "display_name": "Regex",
+    "loading_order": 12,
+    "requires": [],
+    "optional": [],
+    "js": "index.js",
+    "css": "style.css",
+    "author": "kingbri",
+    "version": "1.0.0",
+    "homePage": "https://github.com/SillyTavern/SillyTavern"
+}

public/scripts/extensions/regex/scriptTemplate.html

@@ -0,0 +1,12 @@
+<div class="regex-script-label flex-container flex-grow">
+    <!-- NOTE: Overflow needs to be handled here! -->
+    <div class="regex_script_name flex-grow"></div>
+    <div class="flex-container">
+        <div class="edit_existing_regex menu_button">
+            <i class="fa-solid fa-pencil"></i>
+        </div>
+        <div class="delete_regex menu_button">
+            <i class="fa-solid fa-trash"></i>
+        </div>
+    </div>
+</div>

public/scripts/extensions/regex/style.css

@@ -0,0 +1,37 @@
+.align-start {
+    align-items: start;
+}
+
+.regex_settings .menu_button {
+    width: fit-content;
+    display: flex;
+    gap: 10px;
+    flex-direction: row;
+}
+
+.align-center {
+    align-items: center;
+}
+
+.regex-script-container {
+    margin-top: 10px;
+    margin-bottom: 10px;
+}
+
+.regex-script-label {
+    align-items: center;
+    border: 1px solid rgba(128, 128, 128, 0.5);
+    border-radius: 10px;
+    padding: 0 5px;
+    margin-top: 1px;
+    margin-bottom: 1px;
+    flex-wrap: nowrap;
+}
+
+.align-self-center {
+    align-self: center;
+}
+
+.flex-grow {
+    flex-grow: 1;
+}