Merge remote-tracking branch 'upstream/staging' into staging

public/scripts/extensions.js

@@ -47,8 +47,6 @@ export function saveMetadataDebounced() {
     }, 1000);
 }
 
-export const extensionsHandlebars = Handlebars.create();
-
 /**
  * Provides an ability for extensions to render HTML templates.
  * Templates sanitation and localization is forced.
@@ -61,40 +59,6 @@ export function renderExtensionTemplate(extensionName, templateId, templateData
     return renderTemplate(`scripts/extensions/${extensionName}/${templateId}.html`, templateData, sanitize, localize, true);
 }
 
-/**
- * Registers a Handlebars helper for use in extensions.
- * @param {string} name Handlebars helper name
- * @param {function} helper Handlebars helper function
- */
-export function registerExtensionHelper(name, helper) {
-    extensionsHandlebars.registerHelper(name, helper);
-}
-
-/**
- * Applies handlebars extension helpers to a message.
- * @param {number} messageId Message index in the chat.
- */
-export function processExtensionHelpers(messageId) {
-    const context = getContext();
-    const message = context.chat[messageId];
-
-    if (!message?.mes || typeof message.mes !== 'string') {
-        return;
-    }
-
-    // Don't waste time if there are no mustaches
-    if (!substituteParams(message.mes).includes('{{')) {
-        return;
-    }
-
-    try {
-        const template = extensionsHandlebars.compile(substituteParams(message.mes), { noEscape: true });
-        message.mes = template({});
-    } catch {
-        // Ignore
-    }
-}
-
 // Disables parallel updates
 class ModuleWorkerWrapper {
     constructor(callback) {

public/scripts/extensions/translate/index.js

@@ -12,6 +12,7 @@ import {
 } from '../../../script.js';
 import { extension_settings, getContext } from '../../extensions.js';
 import { secret_state, writeSecret } from '../../secrets.js';
+import { splitRecursive } from '../../utils.js';
 
 export const autoModeOptions = {
     NONE: 'none',
@@ -315,6 +316,28 @@ async function translateProviderBing(text, lang) {
     throw new Error(response.statusText);
 }
 
+/**
+ * Splits text into chunks and translates each chunk separately
+ * @param {string} text Text to translate
+ * @param {string} lang Target language code
+ * @param {(text: string, lang: string) => Promise<string>} translateFn Function to translate a single chunk (must return a Promise)
+ * @param {number} chunkSize Maximum chunk size
+ * @returns {Promise<string>} Translated text
+ */
+async function chunkedTranslate(text, lang, translateFn, chunkSize = 5000) {
+    if (text.length <= chunkSize) {
+        return await translateFn(text, lang);
+    }
+
+    const chunks = splitRecursive(text, chunkSize);
+
+    let result = '';
+    for (const chunk of chunks) {
+        result += await translateFn(chunk, lang);
+    }
+    return result;
+}
+
 /**
  * Translates text using the selected translation provider
  * @param {string} text Text to translate
@@ -331,15 +354,15 @@ async function translate(text, lang) {
             case 'libre':
                 return await translateProviderLibre(text, lang);
             case 'google':
-                return await translateProviderGoogle(text, lang);
+                return await chunkedTranslate(text, lang, translateProviderGoogle, 5000);
             case 'deepl':
                 return await translateProviderDeepl(text, lang);
             case 'deeplx':
-                return await translateProviderDeepLX(text, lang);
+                return await chunkedTranslate(text, lang, translateProviderDeepLX, 1500);
             case 'oneringtranslator':
                 return await translateProviderOneRing(text, lang);
             case 'bing':
-                return await translateProviderBing(text, lang);
+                return await chunkedTranslate(text, lang, translateProviderBing, 1000);
             default:
                 console.error('Unknown translation provider', extension_settings.translate.provider);
                 return text;

public/scripts/openai.js

@@ -21,7 +21,6 @@ import {
     MAX_INJECTION_DEPTH,
     name1,
     name2,
-    replaceBiasMarkup,
     replaceItemizedPromptText,
     resultCheckStatus,
     saveSettingsDebounced,
@@ -443,8 +442,6 @@ function setOpenAIMessages(chat) {
                 content = `${chat[j].name}: ${content}`;
             }
         }
-        content = replaceBiasMarkup(content);
-
         // remove caret return (waste of tokens)
         content = content.replace(/\r/gm, '');
 

public/scripts/slash-commands.js

@@ -20,7 +20,7 @@ import {
     main_api,
     name1,
     reloadCurrentChat,
-    replaceBiasMarkup,
+    removeMacros,
     saveChatConditional,
     sendMessageAsUser,
     sendSystemMessage,
@@ -1260,7 +1260,7 @@ export async function sendMessageAs(args, text) {
 
     // Messages that do nothing but set bias will be hidden from the context
     const bias = extractMessageBias(mesText);
-    const isSystem = replaceBiasMarkup(mesText).trim().length === 0;
+    const isSystem = bias && !removeMacros(mesText).length;
 
     const character = characters.find(x => x.name === name);
     let force_avatar, original_avatar;
@@ -1313,7 +1313,7 @@ export async function sendNarratorMessage(args, text) {
     const name = chat_metadata[NARRATOR_NAME_KEY] || NARRATOR_NAME_DEFAULT;
     // Messages that do nothing but set bias will be hidden from the context
     const bias = extractMessageBias(text);
-    const isSystem = replaceBiasMarkup(text).trim().length === 0;
+    const isSystem = bias && !removeMacros(text).length;
 
     const message = {
         name: name,