From 8f373cf1dc2179d3c088fd8f80d3cfd9bdb575c7 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 1 Nov 2024 21:47:25 +0200 Subject: [PATCH] Macros: refactor with a single replace point --- public/scripts/instruct-mode.js | 23 ++-- public/scripts/macros.js | 217 ++++++++++++++++++-------------- public/scripts/variables.js | 88 +++---------- 3 files changed, 152 insertions(+), 176 deletions(-) diff --git a/public/scripts/instruct-mode.js b/public/scripts/instruct-mode.js index c5b70ac52..e3ee337cb 100644 --- a/public/scripts/instruct-mode.js +++ b/public/scripts/instruct-mode.js @@ -565,16 +565,11 @@ function selectMatchingContextTemplate(name) { /** * Replaces instruct mode macros in the given input string. - * @param {string} input Input string. * @param {Object} env - Map of macro names to the values they'll be substituted with. If the param * values are functions, those functions will be called and their return values are used. - * @returns {string} String with macros replaced. + * @returns {import('./macros.js').Macro[]} Macro objects. */ -export function replaceInstructMacros(input, env) { - if (!input) { - return ''; - } - +export function getInstructMacros(env) { const syspromptMacros = { 'systemPrompt': (power_user.prefer_character_prompt && env.charPrompt ? env.charPrompt : power_user.sysprompt.content), 'defaultSystemPrompt|instructSystem|instructSystemPrompt': power_user.sysprompt.content, @@ -598,20 +593,24 @@ export function replaceInstructMacros(input, env) { 'instructLastInput|instructLastUserPrefix': power_user.instruct.last_input_sequence || power_user.instruct.input_sequence, }; + const macros = []; + for (const [placeholder, value] of Object.entries(instructMacros)) { const regex = new RegExp(`{{(${placeholder})}}`, 'gi'); - input = input.replace(regex, power_user.instruct.enabled ? value : ''); + const replace = () => power_user.instruct.enabled ? value : ''; + macros.push({ regex, replace }); } for (const [placeholder, value] of Object.entries(syspromptMacros)) { const regex = new RegExp(`{{(${placeholder})}}`, 'gi'); - input = input.replace(regex, power_user.sysprompt.enabled ? value : ''); + const replace = () => power_user.sysprompt.enabled ? value : ''; + macros.push({ regex, replace }); } - input = input.replace(/{{exampleSeparator}}/gi, power_user.context.example_separator); - input = input.replace(/{{chatStart}}/gi, power_user.context.chat_start); + macros.push({ regex: /{{exampleSeparator}}/gi, replace: () => power_user.context.example_separator }); + macros.push({ regex: /{{chatStart}}/gi, replace: () => power_user.context.chat_start }); - return input; + return macros; } jQuery(() => { diff --git a/public/scripts/macros.js b/public/scripts/macros.js index 034f5df5e..d9fb5a09b 100644 --- a/public/scripts/macros.js +++ b/public/scripts/macros.js @@ -2,8 +2,14 @@ import { Handlebars, moment, seedrandom, droll } from '../lib.js'; import { chat, chat_metadata, main_api, getMaxContextSize, getCurrentChatId, substituteParams } from '../script.js'; import { timestampToMoment, isDigitsOnly, getStringHash, escapeRegex, uuidv4 } from './utils.js'; import { textgenerationwebui_banned_in_macros } from './textgen-settings.js'; -import { replaceInstructMacros } from './instruct-mode.js'; -import { replaceVariableMacros } from './variables.js'; +import { getInstructMacros } from './instruct-mode.js'; +import { getVariableMacros } from './variables.js'; + +/** + * @typedef Macro + * @property {RegExp} regex - Regular expression to match the macro + * @property {(substring: string, ...args: any[]) => string} replace - Function to replace the macro + */ // Register any macro that you want to leave in the compiled story string Handlebars.registerHelper('trim', () => '{{trim}}'); @@ -261,28 +267,19 @@ function getCurrentSwipeId() { /** * Replaces banned words in macros with an empty string. * Adds them to textgenerationwebui ban list. - * @param {string} inText Text to replace banned words in - * @returns {string} Text without the "banned" macro + * @returns {Macro} */ -function bannedWordsReplace(inText) { - if (!inText) { - return ''; - } - +function getBannedWordsMacro() { const banPattern = /{{banned "(.*)"}}/gi; - - if (main_api == 'textgenerationwebui') { - const bans = inText.matchAll(banPattern); - if (bans) { - for (const banCase of bans) { - console.log('Found banned words in macros: ' + banCase[1]); - textgenerationwebui_banned_in_macros.push(banCase[1]); - } + const banReplace = (match, bannedWord) => { + if (main_api == 'textgenerationwebui') { + console.log('Found banned word in macros: ' + bannedWord); + textgenerationwebui_banned_in_macros.push(bannedWord); } - } + return ''; + }; - inText = inText.replaceAll(banPattern, ''); - return inText; + return { regex: banPattern, replace: banReplace }; } function getTimeSinceLastMessage() { @@ -317,10 +314,13 @@ function getTimeSinceLastMessage() { return 'just now'; } -function randomReplace(input, emptyListPlaceholder = '') { +/** + * Returns a macro that picks a random item from a list. + * @returns {Macro} The random replace macro + */ +function getRandomReplaceMacro() { const randomPattern = /{{random\s?::?([^}]+)}}/gi; - - input = input.replace(randomPattern, (match, listString) => { + const randomReplace = (match, listString) => { // Split on either double colons or comma. If comma is the separator, we are also trimming all items. const list = listString.includes('::') ? listString.split('::') @@ -328,24 +328,29 @@ function randomReplace(input, emptyListPlaceholder = '') { : listString.replace(/\\,/g, '##�COMMA�##').split(',').map(item => item.trim().replace(/##�COMMA�##/g, ',')); if (list.length === 0) { - return emptyListPlaceholder; + return ''; } const rng = seedrandom('added entropy.', { entropy: true }); const randomIndex = Math.floor(rng() * list.length); return list[randomIndex]; - }); - return input; + }; + + return { regex: randomPattern, replace: randomReplace }; } -function pickReplace(input, rawContent, emptyListPlaceholder = '') { - const pickPattern = /{{pick\s?::?([^}]+)}}/gi; - +/** + * Returns a macro that picks a random item from a list with a consistent seed. + * @param {string} rawContent The raw content of the string + * @returns {Macro} The pick replace macro + */ +function getPickReplaceMacro(rawContent) { // We need to have a consistent chat hash, otherwise we'll lose rolls on chat file rename or branch switches // No need to save metadata here - branching and renaming will implicitly do the save for us, and until then loading it like this is consistent const chatIdHash = getChatIdHash(); const rawContentHash = getStringHash(rawContent); - return input.replace(pickPattern, (match, listString, offset) => { + const pickPattern = /{{pick\s?::?([^}]+)}}/gi; + const pickReplace = (match, listString, offset) => { // Split on either double colons or comma. If comma is the separator, we are also trimming all items. const list = listString.includes('::') ? listString.split('::') @@ -353,7 +358,7 @@ function pickReplace(input, rawContent, emptyListPlaceholder = '') { : listString.replace(/\\,/g, '##�COMMA�##').split(',').map(item => item.trim().replace(/##�COMMA�##/g, ',')); if (list.length === 0) { - return emptyListPlaceholder; + return ''; } // We build a hash seed based on: unique chat file, raw content, and the placement inside this content @@ -364,13 +369,17 @@ function pickReplace(input, rawContent, emptyListPlaceholder = '') { const rng = seedrandom(finalSeed); const randomIndex = Math.floor(rng() * list.length); return list[randomIndex]; - }); + }; + + return { regex: pickPattern, replace: pickReplace }; } -function diceRollReplace(input, invalidRollPlaceholder = '') { +/** + * @returns {Macro} The dire roll macro + */ +function getDiceRollMacro() { const rollPattern = /{{roll[ : ]([^}]+)}}/gi; - - return input.replace(rollPattern, (match, matchValue) => { + const rollReplace = (match, matchValue) => { let formula = matchValue.trim(); if (isDigitsOnly(formula)) { @@ -381,32 +390,33 @@ function diceRollReplace(input, invalidRollPlaceholder = '') { if (!isValid) { console.debug(`Invalid roll formula: ${formula}`); - return invalidRollPlaceholder; + return ''; } const result = droll.roll(formula); - return new String(result.total); - }); + if (result === false) return ''; + return String(result.total); + }; + + return { regex: rollPattern, replace: rollReplace }; } /** * Returns the difference between two times. Works with any time format acceptable by moment(). * Can work with {{date}} {{time}} macros - * @param {string} input - The string to replace time difference macros in. - * @returns {string} The string with replaced time difference macros. + * @returns {Macro} The time difference macro */ -function timeDiffReplace(input) { +function getTimeDiffMacro() { const timeDiffPattern = /{{timeDiff::(.*?)::(.*?)}}/gi; - - const output = input.replace(timeDiffPattern, (_match, matchPart1, matchPart2) => { + const timeDiffReplace = (_match, matchPart1, matchPart2) => { const time1 = moment(matchPart1); const time2 = moment(matchPart2); const timeDifference = moment.duration(time1.diff(time2)); return timeDifference.humanize(true); - }); + }; - return output; + return { regex: timeDiffPattern, replace: timeDiffReplace }; } /** @@ -423,72 +433,89 @@ export function evaluateMacros(content, env) { const rawContent = content; - // Legacy non-macro substitutions - content = content.replace(//gi, typeof env.user === 'function' ? env.user() : env.user); - content = content.replace(//gi, typeof env.char === 'function' ? env.char() : env.char); - content = content.replace(//gi, typeof env.char === 'function' ? env.char() : env.char); - content = content.replace(//gi, typeof env.group === 'function' ? env.group() : env.group); - content = content.replace(//gi, typeof env.group === 'function' ? env.group() : env.group); + /** + * Built-ins running before the env variables + * @type {Macro[]} + * */ + const preEnvMacros = [ + // Legacy non-curly macros + { regex: //gi, replace: () => typeof env.user === 'function' ? env.user() : env.user }, + { regex: //gi, replace: () => typeof env.char === 'function' ? env.char() : env.char }, + { regex: //gi, replace: () => typeof env.char === 'function' ? env.char() : env.char }, + { regex: //gi, replace: () => typeof env.group === 'function' ? env.group() : env.group }, + { regex: //gi, replace: () => typeof env.group === 'function' ? env.group() : env.group }, + getDiceRollMacro(), + ...getInstructMacros(env), + ...getVariableMacros(), + { regex: /{{newline}}/gi, replace: () => '\n' }, + { regex: /(?:\r?\n)*{{trim}}(?:\r?\n)*/gi, replace: () => '' }, + { regex: /{{noop}}/gi, replace: () => '' }, + { regex: /{{input}}/gi, replace: () => String($('#send_textarea').val()) }, + ]; - // Short circuit if there are no macros - if (!content.includes('{{')) { - return content; - } - - content = diceRollReplace(content); - content = replaceInstructMacros(content, env); - content = replaceVariableMacros(content); - content = content.replace(/{{newline}}/gi, '\n'); - content = content.replace(/(?:\r?\n)*{{trim}}(?:\r?\n)*/gi, ''); - content = content.replace(/{{noop}}/gi, ''); - content = content.replace(/{{input}}/gi, () => String($('#send_textarea').val())); + /** + * Built-ins running after the env variables + * @type {Macro[]} + */ + const postEnvMacros = [ + { regex: /{{maxPrompt}}/gi, replace: () => String(getMaxContextSize()) }, + { regex: /{{lastMessage}}/gi, replace: () => getLastMessage() }, + { regex: /{{lastMessageId}}/gi, replace: () => String(getLastMessageId() ?? '') }, + { regex: /{{lastUserMessage}}/gi, replace: () => getLastUserMessage() }, + { regex: /{{lastCharMessage}}/gi, replace: () => getLastCharMessage() }, + { regex: /{{firstIncludedMessageId}}/gi, replace: () => String(getFirstIncludedMessageId() ?? '') }, + { regex: /{{lastSwipeId}}/gi, replace: () => String(getLastSwipeId() ?? '') }, + { regex: /{{currentSwipeId}}/gi, replace: () => String(getCurrentSwipeId() ?? '') }, + { regex: /{{reverse:(.+?)}}/gi, replace: (_, str) => Array.from(str).reverse().join('') }, + { regex: /\{\{\/\/([\s\S]*?)\}\}/gm, replace: () => '' }, + { regex: /{{time}}/gi, replace: () => moment().format('LT') }, + { regex: /{{date}}/gi, replace: () => moment().format('LL') }, + { regex: /{{weekday}}/gi, replace: () => moment().format('dddd') }, + { regex: /{{isotime}}/gi, replace: () => moment().format('HH:mm') }, + { regex: /{{isodate}}/gi, replace: () => moment().format('YYYY-MM-DD') }, + { regex: /{{datetimeformat +([^}]*)}}/gi, replace: (_, format) => moment().format(format) }, + { regex: /{{idle_duration}}/gi, replace: () => getTimeSinceLastMessage() }, + { regex: /{{time_UTC([-+]\d+)}}/gi, replace: (_, offset) => moment().utc().utcOffset(parseInt(offset, 10)).format('LT') }, + getTimeDiffMacro(), + getBannedWordsMacro(), + getRandomReplaceMacro(), + getPickReplaceMacro(rawContent), + ]; // Add all registered macros to the env object - const nonce = uuidv4(); MacrosParser.populateEnv(env); + const nonce = uuidv4(); + const envMacros = []; // Substitute passed-in variables for (const varName in env) { if (!Object.hasOwn(env, varName)) continue; - content = content.replace(new RegExp(`{{${escapeRegex(varName)}}}`, 'gi'), () => { + const envRegex = new RegExp(`{{${escapeRegex(varName)}}}`, 'gi'); + const envReplace = () => { const param = env[varName]; const value = MacrosParser.sanitizeMacroValue(typeof param === 'function' ? param(nonce) : param); return value; - }); + }; + + envMacros.push({ regex: envRegex, replace: envReplace }); } - content = content.replace(/{{maxPrompt}}/gi, () => String(getMaxContextSize())); - content = content.replace(/{{lastMessage}}/gi, () => getLastMessage()); - content = content.replace(/{{lastMessageId}}/gi, () => String(getLastMessageId() ?? '')); - content = content.replace(/{{lastUserMessage}}/gi, () => getLastUserMessage()); - content = content.replace(/{{lastCharMessage}}/gi, () => getLastCharMessage()); - content = content.replace(/{{firstIncludedMessageId}}/gi, () => String(getFirstIncludedMessageId() ?? '')); - content = content.replace(/{{lastSwipeId}}/gi, () => String(getLastSwipeId() ?? '')); - content = content.replace(/{{currentSwipeId}}/gi, () => String(getCurrentSwipeId() ?? '')); - content = content.replace(/{{reverse:(.+?)}}/gi, (_, str) => Array.from(str).reverse().join('')); + const macros = [...preEnvMacros, ...envMacros, ...postEnvMacros]; - content = content.replace(/\{\{\/\/([\s\S]*?)\}\}/gm, ''); + for (const macro of macros) { + // Stop if the content is empty + if (!content) { + break; + } - content = content.replace(/{{time}}/gi, () => moment().format('LT')); - content = content.replace(/{{date}}/gi, () => moment().format('LL')); - content = content.replace(/{{weekday}}/gi, () => moment().format('dddd')); - content = content.replace(/{{isotime}}/gi, () => moment().format('HH:mm')); - content = content.replace(/{{isodate}}/gi, () => moment().format('YYYY-MM-DD')); + // Short-circuit if no curly braces are found + if (!macro.regex.source.startsWith('<') && !content.includes('{{')) { + break; + } + + content = content.replace(macro.regex, macro.replace); + } - content = content.replace(/{{datetimeformat +([^}]*)}}/gi, (_, format) => { - const formattedTime = moment().format(format); - return formattedTime; - }); - content = content.replace(/{{idle_duration}}/gi, () => getTimeSinceLastMessage()); - content = content.replace(/{{time_UTC([-+]\d+)}}/gi, (_, offset) => { - const utcOffset = parseInt(offset, 10); - const utcTime = moment().utc().utcOffset(utcOffset).format('LT'); - return utcTime; - }); - content = timeDiffReplace(content); - content = bannedWordsReplace(content); - content = randomReplace(content); - content = pickReplace(content, rawContent); return content; } diff --git a/public/scripts/variables.js b/public/scripts/variables.js index ce8c76578..f871b030c 100644 --- a/public/scripts/variables.js +++ b/public/scripts/variables.js @@ -223,85 +223,34 @@ export function resolveVariable(name, scope = null) { return name; } -export function replaceVariableMacros(input) { - const lines = input.split('\n'); - - for (let i = 0; i < lines.length; i++) { - let line = lines[i]; - - // Skip lines without macros - if (!line || !line.includes('{{')) { - continue; - } - +/** + * @returns {import('./macros.js').Macro[]} + */ +export function getVariableMacros() { + const macros = [ // Replace {{getvar::name}} with the value of the variable name - line = line.replace(/{{getvar::([^}]+)}}/gi, (_, name) => { - name = name.trim(); - return getLocalVariable(name); - }); - + { regex: /{{getvar::([^}]+)}}/gi, replace: (_, name) => getLocalVariable(name.trim()) }, // Replace {{setvar::name::value}} with empty string and set the variable name to value - line = line.replace(/{{setvar::([^:]+)::([^}]+)}}/gi, (_, name, value) => { - name = name.trim(); - setLocalVariable(name, value); - return ''; - }); - + { regex: /{{setvar::([^:]+)::([^}]+)}}/gi, replace: (_, name, value) => { setLocalVariable(name.trim(), value); return ''; } }, // Replace {{addvar::name::value}} with empty string and add value to the variable value - line = line.replace(/{{addvar::([^:]+)::([^}]+)}}/gi, (_, name, value) => { - name = name.trim(); - addLocalVariable(name, value); - return ''; - }); - + { regex: /{{addvar::([^:]+)::([^}]+)}}/gi, replace: (_, name, value) => { addLocalVariable(name.trim(), value); return ''; } }, // Replace {{incvar::name}} with empty string and increment the variable name by 1 - line = line.replace(/{{incvar::([^}]+)}}/gi, (_, name) => { - name = name.trim(); - return incrementLocalVariable(name); - }); - + { regex: /{{incvar::([^}]+)}}/gi, replace: (_, name) => incrementLocalVariable(name.trim()) }, // Replace {{decvar::name}} with empty string and decrement the variable name by 1 - line = line.replace(/{{decvar::([^}]+)}}/gi, (_, name) => { - name = name.trim(); - return decrementLocalVariable(name); - }); - + { regex: /{{decvar::([^}]+)}}/gi, replace: (_, name) => decrementLocalVariable(name.trim()) }, // Replace {{getglobalvar::name}} with the value of the global variable name - line = line.replace(/{{getglobalvar::([^}]+)}}/gi, (_, name) => { - name = name.trim(); - return getGlobalVariable(name); - }); - + { regex: /{{getglobalvar::([^}]+)}}/gi, replace: (_, name) => getGlobalVariable(name.trim()) }, // Replace {{setglobalvar::name::value}} with empty string and set the global variable name to value - line = line.replace(/{{setglobalvar::([^:]+)::([^}]+)}}/gi, (_, name, value) => { - name = name.trim(); - setGlobalVariable(name, value); - return ''; - }); - + { regex: /{{setglobalvar::([^:]+)::([^}]+)}}/gi, replace: (_, name, value) => { setGlobalVariable(name.trim(), value); return ''; } }, // Replace {{addglobalvar::name::value}} with empty string and add value to the global variable value - line = line.replace(/{{addglobalvar::([^:]+)::([^}]+)}}/gi, (_, name, value) => { - name = name.trim(); - addGlobalVariable(name, value); - return ''; - }); - + { regex: /{{addglobalvar::([^:]+)::([^}]+)}}/gi, replace: (_, name, value) => { addGlobalVariable(name.trim(), value); return ''; } }, // Replace {{incglobalvar::name}} with empty string and increment the global variable name by 1 - line = line.replace(/{{incglobalvar::([^}]+)}}/gi, (_, name) => { - name = name.trim(); - return incrementGlobalVariable(name); - }); - + { regex: /{{incglobalvar::([^}]+)}}/gi, replace: (_, name) => incrementGlobalVariable(name.trim()) }, // Replace {{decglobalvar::name}} with empty string and decrement the global variable name by 1 - line = line.replace(/{{decglobalvar::([^}]+)}}/gi, (_, name) => { - name = name.trim(); - return decrementGlobalVariable(name); - }); + { regex: /{{decglobalvar::([^}]+)}}/gi, replace: (_, name) => decrementGlobalVariable(name.trim()) }, + ]; - lines[i] = line; - } - - return lines.join('\n'); + return macros; } async function listVariablesCallback(args) { @@ -2148,7 +2097,8 @@ export function registerVariableCommands() { callback: sortArrayObjectCallback, returns: 'the sorted list or dictionary keys', namedArgumentList: [ - SlashCommandNamedArgument.fromProps({ name: 'keysort', + SlashCommandNamedArgument.fromProps({ + name: 'keysort', description: 'whether to sort by key or value; ignored for lists', typeList: [ARGUMENT_TYPE.BOOLEAN], enumList: ['true', 'false'],