Macros: refactor with a single replace point

public/scripts/instruct-mode.js

@@ -565,16 +565,11 @@ function selectMatchingContextTemplate(name) {
 
 /**
  * Replaces instruct mode macros in the given input string.
- * @param {string} input Input string.
  * @param {Object<string, *>} env - Map of macro names to the values they'll be substituted with. If the param
  * values are functions, those functions will be called and their return values are used.
- * @returns {string} String with macros replaced.
+ * @returns {import('./macros.js').Macro[]} Macro objects.
  */
-export function replaceInstructMacros(input, env) {
+export function getInstructMacros(env) {
-    if (!input) {
-        return '';
-    }
-
     const syspromptMacros = {
         'systemPrompt': (power_user.prefer_character_prompt && env.charPrompt ? env.charPrompt : power_user.sysprompt.content),
         'defaultSystemPrompt|instructSystem|instructSystemPrompt': power_user.sysprompt.content,
@@ -598,20 +593,24 @@ export function replaceInstructMacros(input, env) {
         'instructLastInput|instructLastUserPrefix': power_user.instruct.last_input_sequence || power_user.instruct.input_sequence,
     };
 
+    const macros = [];
+
     for (const [placeholder, value] of Object.entries(instructMacros)) {
         const regex = new RegExp(`{{(${placeholder})}}`, 'gi');
-        input = input.replace(regex, power_user.instruct.enabled ? value : '');
+        const replace = () => power_user.instruct.enabled ? value : '';
+        macros.push({ regex, replace });
     }
 
     for (const [placeholder, value] of Object.entries(syspromptMacros)) {
         const regex = new RegExp(`{{(${placeholder})}}`, 'gi');
-        input = input.replace(regex, power_user.sysprompt.enabled ? value : '');
+        const replace = () => power_user.sysprompt.enabled ? value : '';
+        macros.push({ regex, replace });
     }
 
-    input = input.replace(/{{exampleSeparator}}/gi, power_user.context.example_separator);
+    macros.push({ regex: /{{exampleSeparator}}/gi, replace: () => power_user.context.example_separator });
-    input = input.replace(/{{chatStart}}/gi, power_user.context.chat_start);
+    macros.push({ regex: /{{chatStart}}/gi, replace: () => power_user.context.chat_start });
 
-    return input;
+    return macros;
 }
 
 jQuery(() => {

public/scripts/macros.js

@@ -2,8 +2,14 @@ import { Handlebars, moment, seedrandom, droll } from '../lib.js';
 import { chat, chat_metadata, main_api, getMaxContextSize, getCurrentChatId, substituteParams } from '../script.js';
 import { timestampToMoment, isDigitsOnly, getStringHash, escapeRegex, uuidv4 } from './utils.js';
 import { textgenerationwebui_banned_in_macros } from './textgen-settings.js';
-import { replaceInstructMacros } from './instruct-mode.js';
+import { getInstructMacros } from './instruct-mode.js';
-import { replaceVariableMacros } from './variables.js';
+import { getVariableMacros } from './variables.js';
+
+/**
+ * @typedef Macro
+ * @property {RegExp} regex - Regular expression to match the macro
+ * @property {(substring: string, ...args: any[]) => string} replace - Function to replace the macro
+ */
 
 // Register any macro that you want to leave in the compiled story string
 Handlebars.registerHelper('trim', () => '{{trim}}');
@@ -261,28 +267,19 @@ function getCurrentSwipeId() {
 /**
  * Replaces banned words in macros with an empty string.
  * Adds them to textgenerationwebui ban list.
- * @param {string} inText Text to replace banned words in
+ * @returns {Macro}
- * @returns {string} Text without the "banned" macro
  */
-function bannedWordsReplace(inText) {
+function getBannedWordsMacro() {
-    if (!inText) {
-        return '';
-    }
-
     const banPattern = /{{banned "(.*)"}}/gi;
-
+    const banReplace = (match, bannedWord) => {
         if (main_api == 'textgenerationwebui') {
-        const bans = inText.matchAll(banPattern);
+            console.log('Found banned word in macros: ' + bannedWord);
-        if (bans) {
+            textgenerationwebui_banned_in_macros.push(bannedWord);
-            for (const banCase of bans) {
-                console.log('Found banned words in macros: ' + banCase[1]);
-                textgenerationwebui_banned_in_macros.push(banCase[1]);
-            }
-        }
         }
+        return '';
+    };
 
-    inText = inText.replaceAll(banPattern, '');
+    return { regex: banPattern, replace: banReplace };
-    return inText;
 }
 
 function getTimeSinceLastMessage() {
@@ -317,10 +314,13 @@ function getTimeSinceLastMessage() {
     return 'just now';
 }
 
-function randomReplace(input, emptyListPlaceholder = '') {
+/**
+ * Returns a macro that picks a random item from a list.
+ * @returns {Macro} The random replace macro
+ */
+function getRandomReplaceMacro() {
     const randomPattern = /{{random\s?::?([^}]+)}}/gi;
-
+    const randomReplace = (match, listString) => {
-    input = input.replace(randomPattern, (match, listString) => {
         // Split on either double colons or comma. If comma is the separator, we are also trimming all items.
         const list = listString.includes('::')
             ? listString.split('::')
@@ -328,24 +328,29 @@ function randomReplace(input, emptyListPlaceholder = '') {
             : listString.replace(/\\,/g, '##<23>COMMA<4D>##').split(',').map(item => item.trim().replace(/##<23>COMMA<4D>##/g, ','));
 
         if (list.length === 0) {
-            return emptyListPlaceholder;
+            return '';
         }
         const rng = seedrandom('added entropy.', { entropy: true });
         const randomIndex = Math.floor(rng() * list.length);
         return list[randomIndex];
-    });
+    };
-    return input;
+
+    return { regex: randomPattern, replace: randomReplace };
 }
 
-function pickReplace(input, rawContent, emptyListPlaceholder = '') {
+/**
-    const pickPattern = /{{pick\s?::?([^}]+)}}/gi;
+ * Returns a macro that picks a random item from a list with a consistent seed.
-
+ * @param {string} rawContent The raw content of the string
+ * @returns {Macro} The pick replace macro
+ */
+function getPickReplaceMacro(rawContent) {
     // We need to have a consistent chat hash, otherwise we'll lose rolls on chat file rename or branch switches
     // No need to save metadata here - branching and renaming will implicitly do the save for us, and until then loading it like this is consistent
     const chatIdHash = getChatIdHash();
     const rawContentHash = getStringHash(rawContent);
 
-    return input.replace(pickPattern, (match, listString, offset) => {
+    const pickPattern = /{{pick\s?::?([^}]+)}}/gi;
+    const pickReplace = (match, listString, offset) => {
         // Split on either double colons or comma. If comma is the separator, we are also trimming all items.
         const list = listString.includes('::')
             ? listString.split('::')
@@ -353,7 +358,7 @@ function pickReplace(input, rawContent, emptyListPlaceholder = '') {
             : listString.replace(/\\,/g, '##<23>COMMA<4D>##').split(',').map(item => item.trim().replace(/##<23>COMMA<4D>##/g, ','));
 
         if (list.length === 0) {
-            return emptyListPlaceholder;
+            return '';
         }
 
         // We build a hash seed based on: unique chat file, raw content, and the placement inside this content
@@ -364,13 +369,17 @@ function pickReplace(input, rawContent, emptyListPlaceholder = '') {
         const rng = seedrandom(finalSeed);
         const randomIndex = Math.floor(rng() * list.length);
         return list[randomIndex];
-    });
+    };
+
+    return { regex: pickPattern, replace: pickReplace };
 }
 
-function diceRollReplace(input, invalidRollPlaceholder = '') {
+/**
+ * @returns {Macro} The dire roll macro
+ */
+function getDiceRollMacro() {
     const rollPattern = /{{roll[ : ]([^}]+)}}/gi;
-
+    const rollReplace = (match, matchValue) => {
-    return input.replace(rollPattern, (match, matchValue) => {
         let formula = matchValue.trim();
 
         if (isDigitsOnly(formula)) {
@@ -381,32 +390,33 @@ function diceRollReplace(input, invalidRollPlaceholder = '') {
 
         if (!isValid) {
             console.debug(`Invalid roll formula: ${formula}`);
-            return invalidRollPlaceholder;
+            return '';
         }
 
         const result = droll.roll(formula);
-        return new String(result.total);
+        if (result === false) return '';
-    });
+        return String(result.total);
+    };
+
+    return { regex: rollPattern, replace: rollReplace };
 }
 
 /**
  * Returns the difference between two times. Works with any time format acceptable by moment().
  * Can work with {{date}} {{time}} macros
- * @param {string} input - The string to replace time difference macros in.
+ * @returns {Macro} The time difference macro
- * @returns {string} The string with replaced time difference macros.
  */
-function timeDiffReplace(input) {
+function getTimeDiffMacro() {
     const timeDiffPattern = /{{timeDiff::(.*?)::(.*?)}}/gi;
-
+    const timeDiffReplace = (_match, matchPart1, matchPart2) => {
-    const output = input.replace(timeDiffPattern, (_match, matchPart1, matchPart2) => {
         const time1 = moment(matchPart1);
         const time2 = moment(matchPart2);
 
         const timeDifference = moment.duration(time1.diff(time2));
         return timeDifference.humanize(true);
-    });
+    };
 
-    return output;
+    return { regex: timeDiffPattern, replace: timeDiffReplace };
 }
 
 /**
@@ -423,72 +433,89 @@ export function evaluateMacros(content, env) {
 
     const rawContent = content;
 
-    // Legacy non-macro substitutions
+    /**
-    content = content.replace(/<USER>/gi, typeof env.user === 'function' ? env.user() : env.user);
+     * Built-ins running before the env variables
-    content = content.replace(/<BOT>/gi, typeof env.char === 'function' ? env.char() : env.char);
+     * @type {Macro[]}
-    content = content.replace(/<CHAR>/gi, typeof env.char === 'function' ? env.char() : env.char);
+     * */
-    content = content.replace(/<CHARIFNOTGROUP>/gi, typeof env.group === 'function' ? env.group() : env.group);
+    const preEnvMacros = [
-    content = content.replace(/<GROUP>/gi, typeof env.group === 'function' ? env.group() : env.group);
+        // Legacy non-curly macros
+        { regex: /<USER>/gi, replace: () => typeof env.user === 'function' ? env.user() : env.user },
+        { regex: /<BOT>/gi, replace: () => typeof env.char === 'function' ? env.char() : env.char },
+        { regex: /<CHAR>/gi, replace: () => typeof env.char === 'function' ? env.char() : env.char },
+        { regex: /<CHARIFNOTGROUP>/gi, replace: () => typeof env.group === 'function' ? env.group() : env.group },
+        { regex: /<GROUP>/gi, replace: () => typeof env.group === 'function' ? env.group() : env.group },
+        getDiceRollMacro(),
+        ...getInstructMacros(env),
+        ...getVariableMacros(),
+        { regex: /{{newline}}/gi, replace: () => '\n' },
+        { regex: /(?:\r?\n)*{{trim}}(?:\r?\n)*/gi, replace: () => '' },
+        { regex: /{{noop}}/gi, replace: () => '' },
+        { regex: /{{input}}/gi, replace: () => String($('#send_textarea').val()) },
+    ];
 
-    // Short circuit if there are no macros
+    /**
-    if (!content.includes('{{')) {
+     * Built-ins running after the env variables
-        return content;
+     * @type {Macro[]}
-    }
+    */
-
+    const postEnvMacros = [
-    content = diceRollReplace(content);
+        { regex: /{{maxPrompt}}/gi, replace: () => String(getMaxContextSize()) },
-    content = replaceInstructMacros(content, env);
+        { regex: /{{lastMessage}}/gi, replace: () => getLastMessage() },
-    content = replaceVariableMacros(content);
+        { regex: /{{lastMessageId}}/gi, replace: () => String(getLastMessageId() ?? '') },
-    content = content.replace(/{{newline}}/gi, '\n');
+        { regex: /{{lastUserMessage}}/gi, replace: () => getLastUserMessage() },
-    content = content.replace(/(?:\r?\n)*{{trim}}(?:\r?\n)*/gi, '');
+        { regex: /{{lastCharMessage}}/gi, replace: () => getLastCharMessage() },
-    content = content.replace(/{{noop}}/gi, '');
+        { regex: /{{firstIncludedMessageId}}/gi, replace: () => String(getFirstIncludedMessageId() ?? '') },
-    content = content.replace(/{{input}}/gi, () => String($('#send_textarea').val()));
+        { regex: /{{lastSwipeId}}/gi, replace: () => String(getLastSwipeId() ?? '') },
+        { regex: /{{currentSwipeId}}/gi, replace: () => String(getCurrentSwipeId() ?? '') },
+        { regex: /{{reverse:(.+?)}}/gi, replace: (_, str) => Array.from(str).reverse().join('') },
+        { regex: /\{\{\/\/([\s\S]*?)\}\}/gm, replace: () => '' },
+        { regex: /{{time}}/gi, replace: () => moment().format('LT') },
+        { regex: /{{date}}/gi, replace: () => moment().format('LL') },
+        { regex: /{{weekday}}/gi, replace: () => moment().format('dddd') },
+        { regex: /{{isotime}}/gi, replace: () => moment().format('HH:mm') },
+        { regex: /{{isodate}}/gi, replace: () => moment().format('YYYY-MM-DD') },
+        { regex: /{{datetimeformat +([^}]*)}}/gi, replace: (_, format) => moment().format(format) },
+        { regex: /{{idle_duration}}/gi, replace: () => getTimeSinceLastMessage() },
+        { regex: /{{time_UTC([-+]\d+)}}/gi, replace: (_, offset) => moment().utc().utcOffset(parseInt(offset, 10)).format('LT') },
+        getTimeDiffMacro(),
+        getBannedWordsMacro(),
+        getRandomReplaceMacro(),
+        getPickReplaceMacro(rawContent),
+    ];
 
     // Add all registered macros to the env object
-    const nonce = uuidv4();
     MacrosParser.populateEnv(env);
+    const nonce = uuidv4();
+    const envMacros = [];
 
     // Substitute passed-in variables
     for (const varName in env) {
         if (!Object.hasOwn(env, varName)) continue;
 
-        content = content.replace(new RegExp(`{{${escapeRegex(varName)}}}`, 'gi'), () => {
+        const envRegex = new RegExp(`{{${escapeRegex(varName)}}}`, 'gi');
+        const envReplace = () => {
             const param = env[varName];
             const value = MacrosParser.sanitizeMacroValue(typeof param === 'function' ? param(nonce) : param);
             return value;
-        });
+        };
+
+        envMacros.push({ regex: envRegex, replace: envReplace });
     }
 
-    content = content.replace(/{{maxPrompt}}/gi, () => String(getMaxContextSize()));
+    const macros = [...preEnvMacros, ...envMacros, ...postEnvMacros];
-    content = content.replace(/{{lastMessage}}/gi, () => getLastMessage());
-    content = content.replace(/{{lastMessageId}}/gi, () => String(getLastMessageId() ?? ''));
-    content = content.replace(/{{lastUserMessage}}/gi, () => getLastUserMessage());
-    content = content.replace(/{{lastCharMessage}}/gi, () => getLastCharMessage());
-    content = content.replace(/{{firstIncludedMessageId}}/gi, () => String(getFirstIncludedMessageId() ?? ''));
-    content = content.replace(/{{lastSwipeId}}/gi, () => String(getLastSwipeId() ?? ''));
-    content = content.replace(/{{currentSwipeId}}/gi, () => String(getCurrentSwipeId() ?? ''));
-    content = content.replace(/{{reverse:(.+?)}}/gi, (_, str) => Array.from(str).reverse().join(''));
 
-    content = content.replace(/\{\{\/\/([\s\S]*?)\}\}/gm, '');
+    for (const macro of macros) {
+        // Stop if the content is empty
+        if (!content) {
+            break;
+        }
 
-    content = content.replace(/{{time}}/gi, () => moment().format('LT'));
+        // Short-circuit if no curly braces are found
-    content = content.replace(/{{date}}/gi, () => moment().format('LL'));
+        if (!macro.regex.source.startsWith('<') && !content.includes('{{')) {
-    content = content.replace(/{{weekday}}/gi, () => moment().format('dddd'));
+            break;
-    content = content.replace(/{{isotime}}/gi, () => moment().format('HH:mm'));
+        }
-    content = content.replace(/{{isodate}}/gi, () => moment().format('YYYY-MM-DD'));
+
+        content = content.replace(macro.regex, macro.replace);
+    }
 
-    content = content.replace(/{{datetimeformat +([^}]*)}}/gi, (_, format) => {
-        const formattedTime = moment().format(format);
-        return formattedTime;
-    });
-    content = content.replace(/{{idle_duration}}/gi, () => getTimeSinceLastMessage());
-    content = content.replace(/{{time_UTC([-+]\d+)}}/gi, (_, offset) => {
-        const utcOffset = parseInt(offset, 10);
-        const utcTime = moment().utc().utcOffset(utcOffset).format('LT');
-        return utcTime;
-    });
-    content = timeDiffReplace(content);
-    content = bannedWordsReplace(content);
-    content = randomReplace(content);
-    content = pickReplace(content, rawContent);
     return content;
 }

public/scripts/variables.js

@@ -223,85 +223,34 @@ export function resolveVariable(name, scope = null) {
     return name;
 }
 
-export function replaceVariableMacros(input) {
+/**
-    const lines = input.split('\n');
+ * @returns {import('./macros.js').Macro[]}
-
+ */
-    for (let i = 0; i < lines.length; i++) {
+export function getVariableMacros() {
-        let line = lines[i];
+    const macros = [
-
-        // Skip lines without macros
-        if (!line || !line.includes('{{')) {
-            continue;
-        }
-
         // Replace {{getvar::name}} with the value of the variable name
-        line = line.replace(/{{getvar::([^}]+)}}/gi, (_, name) => {
+        { regex: /{{getvar::([^}]+)}}/gi, replace: (_, name) => getLocalVariable(name.trim()) },
-            name = name.trim();
-            return getLocalVariable(name);
-        });
-
         // Replace {{setvar::name::value}} with empty string and set the variable name to value
-        line = line.replace(/{{setvar::([^:]+)::([^}]+)}}/gi, (_, name, value) => {
+        { regex: /{{setvar::([^:]+)::([^}]+)}}/gi, replace: (_, name, value) => { setLocalVariable(name.trim(), value); return ''; } },
-            name = name.trim();
-            setLocalVariable(name, value);
-            return '';
-        });
-
         // Replace {{addvar::name::value}} with empty string and add value to the variable value
-        line = line.replace(/{{addvar::([^:]+)::([^}]+)}}/gi, (_, name, value) => {
+        { regex: /{{addvar::([^:]+)::([^}]+)}}/gi, replace: (_, name, value) => { addLocalVariable(name.trim(), value); return ''; } },
-            name = name.trim();
-            addLocalVariable(name, value);
-            return '';
-        });
-
         // Replace {{incvar::name}} with empty string and increment the variable name by 1
-        line = line.replace(/{{incvar::([^}]+)}}/gi, (_, name) => {
+        { regex: /{{incvar::([^}]+)}}/gi, replace: (_, name) => incrementLocalVariable(name.trim()) },
-            name = name.trim();
-            return incrementLocalVariable(name);
-        });
-
         // Replace {{decvar::name}} with empty string and decrement the variable name by 1
-        line = line.replace(/{{decvar::([^}]+)}}/gi, (_, name) => {
+        { regex: /{{decvar::([^}]+)}}/gi, replace: (_, name) => decrementLocalVariable(name.trim()) },
-            name = name.trim();
-            return decrementLocalVariable(name);
-        });
-
         // Replace {{getglobalvar::name}} with the value of the global variable name
-        line = line.replace(/{{getglobalvar::([^}]+)}}/gi, (_, name) => {
+        { regex: /{{getglobalvar::([^}]+)}}/gi, replace: (_, name) => getGlobalVariable(name.trim()) },
-            name = name.trim();
-            return getGlobalVariable(name);
-        });
-
         // Replace {{setglobalvar::name::value}} with empty string and set the global variable name to value
-        line = line.replace(/{{setglobalvar::([^:]+)::([^}]+)}}/gi, (_, name, value) => {
+        { regex: /{{setglobalvar::([^:]+)::([^}]+)}}/gi, replace: (_, name, value) => { setGlobalVariable(name.trim(), value); return ''; } },
-            name = name.trim();
-            setGlobalVariable(name, value);
-            return '';
-        });
-
         // Replace {{addglobalvar::name::value}} with empty string and add value to the global variable value
-        line = line.replace(/{{addglobalvar::([^:]+)::([^}]+)}}/gi, (_, name, value) => {
+        { regex: /{{addglobalvar::([^:]+)::([^}]+)}}/gi, replace: (_, name, value) => { addGlobalVariable(name.trim(), value); return ''; } },
-            name = name.trim();
-            addGlobalVariable(name, value);
-            return '';
-        });
-
         // Replace {{incglobalvar::name}} with empty string and increment the global variable name by 1
-        line = line.replace(/{{incglobalvar::([^}]+)}}/gi, (_, name) => {
+        { regex: /{{incglobalvar::([^}]+)}}/gi, replace: (_, name) => incrementGlobalVariable(name.trim()) },
-            name = name.trim();
-            return incrementGlobalVariable(name);
-        });
-
         // Replace {{decglobalvar::name}} with empty string and decrement the global variable name by 1
-        line = line.replace(/{{decglobalvar::([^}]+)}}/gi, (_, name) => {
+        { regex: /{{decglobalvar::([^}]+)}}/gi, replace: (_, name) => decrementGlobalVariable(name.trim()) },
-            name = name.trim();
+    ];
-            return decrementGlobalVariable(name);
-        });
 
-        lines[i] = line;
+    return macros;
-    }
-
-    return lines.join('\n');
 }
 
 async function listVariablesCallback(args) {
@@ -2148,7 +2097,8 @@ export function registerVariableCommands() {
         callback: sortArrayObjectCallback,
         returns: 'the sorted list or dictionary keys',
         namedArgumentList: [
-            SlashCommandNamedArgument.fromProps({ name: 'keysort',
+            SlashCommandNamedArgument.fromProps({
+                name: 'keysort',
                 description: 'whether to sort by key or value; ignored for lists',
                 typeList: [ARGUMENT_TYPE.BOOLEAN],
                 enumList: ['true', 'false'],