Extensions: Add regex engine

Regex is a method that is commonly used to find and replace parts
of a string using a single pattern. Add support for using regex in
SillyTavern which allows users to dynamically change various aspects
of the chatting experience.

Users are able to choose where a given regex script should apply
(both invasive and non-invasive options!). Invasive options alter
chat history while non-invasive alters markdown display for the
entire chat.

A new variable called {{match}} is added in regex scripts which
substitutes in the found match from the original find regex script.

There is a lot more that can be added to this extension, but for now,
this is enough.

Signed-off-by: kingbri <bdashore3@proton.me>

public/script.js

@@ -161,6 +161,8 @@ import { context_settings, loadContextTemplatesFromSettings } from "./scripts/co
 import { markdownExclusionExt } from "./scripts/showdown-exclusion.js";
 import { NOTE_MODULE_NAME, metadata_keys, setFloatingPrompt, shouldWIAddPrompt } from "./scripts/extensions/floating-prompt/index.js";
 import { deviceInfo } from "./scripts/RossAscends-mods.js";
+import { runRegexScript } from "./scripts/extensions/regex/engine.js";
+import { REGEX_PLACEMENT } from "./scripts/extensions/regex/index.js";
 
 //exporting functions and vars for mods
 export {
@@ -488,6 +490,7 @@ export const event_types = {
     IMPERSONATE_READY: 'impersonate_ready',
     CHAT_CHANGED: 'chat_id_changed',
     GENERATION_STOPPED: 'generation_stopped',
+    SETTINGS_LOADED: 'settings_loaded',
     SETTINGS_UPDATED: 'settings_updated',
     GROUP_UPDATED: 'group_updated',
     MOVABLE_PANELS_RESET: 'movable_panels_reset',
@@ -1108,6 +1111,15 @@ function messageFormatting(mes, ch_name, isSystem, isUser) {
         mes = mes.replaceAll(substituteParams(power_user.user_prompt_bias), "");
     }
 
+    extension_settings.regex.forEach((script) => {
+        if (script.placement.includes(REGEX_PLACEMENT.mdDisplay)) {
+            const regexResult = runRegexScript(script, mes);
+            if (regexResult) {
+                mes = regexResult;
+            }
+        }
+    });
+
     if (power_user.auto_fix_generated_markdown) {
         mes = fixMarkdown(mes);
     }
@@ -2865,6 +2877,15 @@ export function replaceBiasMarkup(str) {
 }
 
 export async function sendMessageAsUser(textareaText, messageBias) {
+    extension_settings.regex.forEach((script) => {
+        if (script.placement.includes(REGEX_PLACEMENT.userInput)) {
+            const regexResult = runRegexScript(script, textareaText);
+            if (regexResult) {
+                textareaText = regexResult;
+            }
+        }
+    });
+
     chat[chat.length] = {};
     chat[chat.length - 1]['name'] = name1;
     chat[chat.length - 1]['is_user'] = true;
@@ -3445,11 +3466,28 @@ function extractMessageFromData(data) {
 }
 
 function cleanUpMessage(getMessage, isImpersonate, displayIncompleteSentences = false) {
-    // Append the user bias first before trimming anything else
+    // Add the prompt bias before anything else
-    if (power_user.user_prompt_bias && power_user.user_prompt_bias.length !== 0) {
+    if (
+        power_user.user_prompt_bias && 
+        !isImpersonate &&
+        power_user.user_prompt_bias.length !== 0
+    ) {
         getMessage = substituteParams(power_user.user_prompt_bias) + getMessage;
     }
 
+    // Regex uses vars, so add before formatting
+    extension_settings.regex.forEach((script) => {
+        if (
+            (script.placement.includes(REGEX_PLACEMENT.aiOutput) && !isImpersonate) ||
+            (script.placement.includes(REGEX_PLACEMENT.userInput) && isImpersonate)
+        ) {
+            const regexResult = runRegexScript(script, getMessage);
+            if (regexResult) {
+                getMessage = regexResult;
+            }
+        }
+    });
+
     if (!displayIncompleteSentences && power_user.trim_sentences) {
         getMessage = end_trim_to_sentence(getMessage, power_user.include_newline);
     }
@@ -4771,6 +4809,8 @@ async function getSettings(type) {
     }
 
     if (!is_checked_colab) isColab();
+
+    eventSource.emit(event_types.SETTINGS_LOADED);
 }
 
 function selectKoboldGuiPreset() {
@@ -4859,13 +4899,26 @@ function setCharacterBlockHeight() {
 function updateMessage(div) {
     const mesBlock = div.closest(".mes_block");
     let text = mesBlock.find(".edit_textarea").val();
+    const mes = chat[this_edit_mes_id];
+
+    extension_settings.regex.forEach((script) => {
+        if (script.runOnEdit && (
+            (script.placement.includes(REGEX_PLACEMENT.aiOutput) && mes.is_name) || 
+            (script.placement.includes(REGEX_PLACEMENT.userInput) && mes.is_user) ||
+            (script.placement.includes(REGEX_PLACEMENT.system) && mes.extra?.type === "narrator")
+        )) {
+            const regexResult = runRegexScript(script, text);
+            if (regexResult) {
+                text = regexResult;
+            }
+        }
+    });
 
     if (power_user.trim_spaces) {
         text = text.trim();
     }
 
     const bias = extractMessageBias(text);
-    const mes = chat[this_edit_mes_id];
     mes["mes"] = text;
     if (mes["swipe_id"] !== undefined) {
         mes["swipes"][mes["swipe_id"]] = text;
@@ -5959,9 +6012,11 @@ async function createOrEditCharacter(e) {
             success: async function (html) {
                 if (chat.length === 1 && !selected_group) {
                     var this_ch_mes = default_ch_mes;
+
                     if ($("#firstmessage_textarea").val() != "") {
                         this_ch_mes = $("#firstmessage_textarea").val();
                     }
+
                     if (
                         this_ch_mes !=
                         $.trim(
@@ -5972,6 +6027,17 @@ async function createOrEditCharacter(e) {
                                 .text()
                         )
                     ) {
+                        // MARK - kingbri: Regex on character greeting message
+                        // May need to be placed somewhere else
+                        extension_settings.regex.forEach((script) => {
+                            if (script.placement.includes(REGEX_PLACEMENT.aiOutput)) {
+                                const regexResult = runRegexScript(script, this_ch_mes);
+                                if (regexResult) {
+                                    this_ch_mes = regexResult
+                                }
+                            }
+                        });
+
                         clearChat();
                         chat.length = 0;
                         chat[0] = {};

public/scripts/extensions.js

@@ -56,6 +56,7 @@ const extension_settings = {
     caption: {},
     expressions: {},
     dice: {},
+    regex: [],
     tts: {},
     sd: {},
     chromadb: {},

public/scripts/extensions/regex/dropdown.html

@@ -0,0 +1,17 @@
+<div class="regex_settings">
+    <div class="inline-drawer">
+        <div class="inline-drawer-toggle inline-drawer-header">
+            <b>Regex</b>
+            <div class="inline-drawer-icon fa-solid fa-circle-chevron-down down"></div>
+        </div>
+        <div class="inline-drawer-content">
+            <div id="open_regex_editor" class="menu_button">
+                <i class="fa-solid fa-pen-to-square"></i>
+                <span>Open Editor</span>
+            </div>
+            <hr />
+            <label>Saved Scripts</label>
+            <div id="saved_regex_scripts" class="flex-container regex-script-container flexFlowColumn"></div>
+        </div>
+    </div>
+</div>

public/scripts/extensions/regex/editor.html

@@ -0,0 +1,84 @@
+<div id="regex_editor_template">
+    <div class="regex_editor">
+        <h3><strong>Regex Editor</strong></h3>
+
+        <div class="flex-container flexFlowColumn">
+            <div class="flex1">
+                <label for="regex_script_name" class="title_restorable">
+                    <small data-i18n="Script Name">Script Name</small>
+                </label>
+                <div>
+                    <input class="regex_script_name text_pole textarea_compact" type="text" maxlength="100" />
+                </div>
+            </div>
+            <div class="flex1">
+                <label for="find_regex" class="title_restorable">
+                    <small data-i18n="Find Regex">Find Regex</small>
+                </label>
+                <div>
+                    <input class="find_regex text_pole textarea_compact" type="text" maxlength="100" />
+                </div>
+            </div>
+            <div class="flex1">
+                <label for="replace_string" class="title_restorable">
+                    <small data-i18n="Replace With">Replace With</small>
+                </label>
+                <div>
+                    <textarea 
+                        class="regex_replace_string text_pole wide100p textarea_compact"
+                        placeholder="Use {{match}} to include the matched text from Find Regex"
+                        rows="2"
+                        maxlength="100"
+                    ></textarea>
+                </div>
+            </div>
+        </div>
+
+        <div class="flex-container">
+            <div class="wi-enter-footer-text flex-container flexFlowColumn flexNoGap align-start">
+                <small>Placement</small>
+                <div>
+                    <label class="checkbox flex-container">
+                        <input type="checkbox" name="replace_position" value="0">
+                        <span data-i18n="Author's Note">Markdown Display</span>
+                    </label>
+                </div>
+                <div>
+                    <label class="checkbox flex-container">
+                        <input type="checkbox" name="replace_position" value="1">
+                        <span data-i18n="Before Char">User Input</span>
+                    </label>
+                </div>
+                <div>
+                    <label class="checkbox flex-container">
+                        <input type="checkbox" name="replace_position" value="2">
+                        <span data-i18n="After Char">AI Output</span>
+                    </label>
+                </div>
+                <div>
+                    <label class="checkbox flex-container">
+                        <input type="checkbox" name="replace_position" value="3">
+                        <span data-i18n="Author's Note">/sys Command</span>
+                    </label>
+                </div>
+                <div>
+                    <label class="checkbox flex-container">
+                        <input type="checkbox" name="replace_position" value="4">
+                        <span data-i18n="Author's Note">/sendas Command</span>
+                    </label>
+                </div>
+            </div>
+            <div class="wi-enter-footer-text flex-container flexFlowColumn flexNoGap align-start">
+                <small>Other Options</small>
+                <label class="checkbox flex-container">
+                    <input type="checkbox" name="disabled" />
+                    <span data-i18n="Disabled">Disabled</span>
+                </label>
+                <label class="checkbox flex-container">
+                    <input type="checkbox" name="run_on_edit" />
+                    <span data-i18n="Run On Edit">Run On Edit</span>
+                </label>
+            </div>
+        </div>
+    </div>
+</div>

public/scripts/extensions/regex/engine.js

@@ -0,0 +1,58 @@
+import { substituteParams } from "../../../script.js";
+export {
+    runRegexScript
+}
+
+// From: https://github.com/IonicaBizau/regex-parser.js/blob/master/lib/index.js
+function regexFromString(input) {
+    // Parse input
+    var m = input.match(/(\/?)(.+)\1([a-z]*)/i);
+
+    // Invalid flags
+    if (m[3] && !/^(?!.*?(.).*?\1)[gmixXsuUAJ]+$/.test(m[3])) {
+        return RegExp(input);
+    }
+
+    // Create the regular expression
+    return new RegExp(m[2], m[3]);
+}
+
+// Runs the provided regex script on the given string
+function runRegexScript(regexScript, rawString) {
+    if (!!(regexScript.disabled)) {
+        return;
+    }
+
+    let match;
+    let newString;
+    const findRegex = regexFromString(regexScript.findRegex);
+    while ((match = findRegex.exec(rawString)) !== null) {
+        const fencedMatch = match[0];
+        const capturedMatch = match[1];
+
+        // TODO: Use substrings for replacement. But not necessary at this time.
+        // A substring is from match.index to match.index + match[0].length or fencedMatch.length
+        const subReplaceString = substituteRegexParams(regexScript.replaceString, { regexMatch: capturedMatch ?? fencedMatch });
+        if (!newString) {
+            newString = rawString.replace(fencedMatch, subReplaceString);
+        } else {
+            newString = newString.replace(fencedMatch, subReplaceString);
+        }
+
+        // If the regex isn't global, break out of the loop
+        if (!findRegex.flags.includes('g')) {
+            break;
+        }
+    }
+
+    return newString;
+}
+
+// Substitutes parameters 
+function substituteRegexParams(rawString, { regexMatch }) {
+    let finalString = rawString;
+    finalString = finalString.replace("{{match}}", regexMatch);
+    finalString = substituteParams(finalString);
+
+    return finalString;
+}

public/scripts/extensions/regex/index.js

@@ -0,0 +1,155 @@
+import { callPopup, eventSource, event_types, saveSettingsDebounced } from "../../../script.js";
+import { extension_settings } from "../../extensions.js";
+import { uuidv4 } from "../../utils.js";
+export { REGEX_PLACEMENT }
+
+const REGEX_PLACEMENT = {
+    mdDisplay: 0,
+    userInput: 1,
+    aiOutput: 2,
+    system: 3,
+    sendas: 4
+}
+
+async function saveRegexScript(regexScript, existingScriptIndex) {
+    // If the script index is undefined or already exists
+    // Don't fire this if editing an existing script
+    if (existingScriptIndex === -1) {
+        if (!regexScript.scriptName) {
+            toastr.error(`Could not save regex: The script name was undefined or empty!`);
+            return;
+        }
+    
+        if (extension_settings.regex.find((e) => e.scriptName === regexScript.scriptName)) {
+            toastr.error(`Could not save regex: The name ${regexScript.scriptName} already exists.`);
+            return;
+        }
+    }
+
+    if (regexScript.placement.length === 0) {
+        toastr.error(`Could not save regex: One placement checkbox must be selected!`);
+        return;
+    }
+
+    if (existingScriptIndex !== -1) {
+        extension_settings.regex[existingScriptIndex] = regexScript;
+    } else {
+        extension_settings.regex.push(regexScript);
+    }
+
+    saveSettingsDebounced();
+    await loadRegexScripts();
+}
+
+async function deleteRegexScript({ existingId }) {
+    let scriptName = $(`#${existingId}`).find('.regex_script_name').text();
+
+    const existingScriptIndex = extension_settings.regex.findIndex((script) => script.scriptName === scriptName);
+    if (!existingScriptIndex || existingScriptIndex !== -1) {
+        extension_settings.regex.splice(existingScriptIndex, 1);
+
+        saveSettingsDebounced();
+        await loadRegexScripts();
+    }
+}
+
+async function loadRegexScripts() {
+    $("#saved_regex_scripts").empty();
+
+    const scriptTemplate = $(await $.get("scripts/extensions/regex/scriptTemplate.html"));
+
+    extension_settings.regex.forEach((script) => {
+        // Have to clone here
+        const scriptHtml = scriptTemplate.clone();
+        scriptHtml.attr('id', uuidv4());
+        scriptHtml.find('.regex_script_name').text(script.scriptName);
+        scriptHtml.find('.edit_existing_regex').on('click', async function() {
+            await onRegexEditorOpenClick(scriptHtml.attr("id"));
+        });
+        scriptHtml.find('.delete_regex').on('click', async function() {
+            await deleteRegexScript({ existingId: scriptHtml.attr("id") });
+        });
+
+        $("#saved_regex_scripts").append(scriptHtml);
+    });
+}
+
+async function onRegexEditorOpenClick(existingId) {
+    const editorHtml = $(await $.get("scripts/extensions/regex/editor.html"));
+
+    // If an ID exists, fill in all the values
+    let existingScriptIndex = -1;
+    if (existingId) {
+        const existingScriptName = $(`#${existingId}`).find('.regex_script_name').text();
+        existingScriptIndex = extension_settings.regex.findIndex((script) => script.scriptName === existingScriptName);
+        if (existingScriptIndex !== -1) {
+            const existingScript = extension_settings.regex[existingScriptIndex];
+            editorHtml.find(`.regex_script_name`).val(existingScript.scriptName);
+            editorHtml.find(`.find_regex`).val(existingScript.findRegex);
+            editorHtml.find(`.regex_replace_string`).val(existingScript.replaceString);
+            editorHtml
+                .find(`input[name="disabled"]`)
+                .prop("checked", existingScript.disabled);
+            editorHtml
+                .find(`input[name="run_on_edit"]`)
+                .prop("checked", existingScript.runOnEdit);
+
+            existingScript.placement.forEach((element) => {
+                editorHtml
+                    .find(`input[name="replace_position"][value="${element}"]`)
+                    .prop("checked", true);
+            });
+        }
+    } else {
+        editorHtml
+            .find(`input[name="run_on_edit"]`)
+            .prop("checked", true);
+
+        editorHtml
+            .find(`input[name="replace_position"][value="0"]`)
+            .prop("checked", true);
+    }
+
+    const popupResult = await callPopup(editorHtml, "confirm", undefined, "Save");
+    if (popupResult) {
+        const newRegexScript = {
+            scriptName: editorHtml.find(".regex_script_name").val(),
+            findRegex: editorHtml.find(".find_regex").val(),
+            replaceString: editorHtml.find(".regex_replace_string").val(),
+            placement:
+                editorHtml
+                    .find(`input[name="replace_position"]`)
+                    .filter(":checked")
+                    .map(function() { return parseInt($(this).val()) })
+                    .get()
+                    .filter((e) => e !== NaN) ?? [],
+            disabled:
+                editorHtml
+                    .find(`input[name="disabled"]`)
+                    .prop("checked"),
+            runOnEdit:
+                editorHtml
+                    .find(`input[name="run_on_edit"]`)
+                    .prop("checked")
+        };
+
+        saveRegexScript(newRegexScript, existingScriptIndex);
+    }
+}
+
+function hookToEvents() {
+    eventSource.on(event_types.SETTINGS_LOADED, async function () {
+        await loadRegexScripts();
+    });
+}
+
+jQuery(async () => {
+    const settingsHtml = await $.get("scripts/extensions/regex/dropdown.html");
+    $("#extensions_settings2").append(settingsHtml);
+    $("#open_regex_editor").on("click", function() {
+        onRegexEditorOpenClick(false);
+    });
+
+    // Listen to event source after 1ms
+    setTimeout(() => hookToEvents(), 1)
+});

public/scripts/extensions/regex/manifest.json

@@ -0,0 +1,11 @@
+{
+    "display_name": "Regex",
+    "loading_order": 12,
+    "requires": [],
+    "optional": [],
+    "js": "index.js",
+    "css": "style.css",
+    "author": "kingbri",
+    "version": "1.0.0",
+    "homePage": "https://github.com/SillyTavern/SillyTavern"
+}

public/scripts/extensions/regex/scriptTemplate.html

@@ -0,0 +1,12 @@
+<div class="regex-script-label flex-container flex-grow">
+    <!-- NOTE: Overflow needs to be handled here! -->
+    <div class="regex_script_name flex-grow"></div>
+    <div class="flex-container">
+        <div class="edit_existing_regex menu_button">
+            <i class="fa-solid fa-pencil"></i>
+        </div>
+        <div class="delete_regex menu_button">
+            <i class="fa-solid fa-trash"></i>
+        </div>
+    </div>
+</div>

public/scripts/extensions/regex/style.css

@@ -0,0 +1,37 @@
+.align-start {
+    align-items: start;
+}
+
+.regex_settings .menu_button {
+    width: fit-content;
+    display: flex;
+    gap: 10px;
+    flex-direction: row;
+}
+
+.align-center {
+    align-items: center;
+}
+
+.regex-script-container {
+    margin-top: 10px;
+    margin-bottom: 10px;
+}
+
+.regex-script-label {
+    align-items: center;
+    border: 1px solid rgba(128, 128, 128, 0.5);
+    border-radius: 10px;
+    padding: 0 5px;
+    margin-top: 1px;
+    margin-bottom: 1px;
+    flex-wrap: nowrap;
+}
+
+.align-self-center {
+    align-self: center;
+}
+
+.flex-grow {
+    flex-grow: 1;
+}

public/scripts/slash-commands.js

@@ -22,6 +22,9 @@ import {
 } from "../script.js";
 import { humanizedDateTime } from "./RossAscends-mods.js";
 import { resetSelectedGroup } from "./group-chats.js";
+import { extension_settings } from "./extensions.js";
+import { runRegexScript } from "./extensions/regex/engine.js";
+import { REGEX_PLACEMENT } from "./extensions/regex/index.js";
 import { chat_styles, power_user } from "./power-user.js";
 export {
     executeSlashCommands,
@@ -218,14 +221,22 @@ async function sendMessageAs(_, text) {
     }
 
     const parts = text.split('\n');
-
     if (parts.length <= 1) {
         toastr.warning('Both character name and message are required. Separate them with a new line.');
         return;
     }
 
     const name = parts.shift().trim();
-    const mesText = parts.join('\n').trim();
+    let mesText = parts.join('\n').trim();
+    extension_settings.regex.forEach((script) => {
+        if (script.placement.includes(REGEX_PLACEMENT.sendas)) {
+            const regexResult = runRegexScript(script, mesText);
+            if (regexResult) {
+                mesText = regexResult;
+            }
+        }
+    });
+
     // Messages that do nothing but set bias will be hidden from the context
     const bias = extractMessageBias(mesText);
     const isSystem = replaceBiasMarkup(mesText).trim().length === 0;
@@ -268,6 +279,15 @@ async function sendNarratorMessage(_, text) {
         return;
     }
 
+    extension_settings.regex.forEach((script) => {
+        if (script.placement.includes(REGEX_PLACEMENT.system)) {
+            const regexResult = runRegexScript(script, text);
+            if (regexResult) {
+                text = regexResult;
+            }
+        }
+    });
+
     const name = chat_metadata[NARRATOR_NAME_KEY] || NARRATOR_NAME_DEFAULT;
     // Messages that do nothing but set bias will be hidden from the context
     const bias = extractMessageBias(text);

public/style.css

@@ -4060,6 +4060,10 @@ toolcool-color-picker {
     justify-content: space-around;
 }
 
+.justifyContentFlexStart {
+    justify-content: flex-start;
+}
+
 .justifyContentFlexEnd {
     justify-content: flex-end;
 }