Add images to quiet prompts if inlining enabled

public/script.js

@@ -2104,16 +2104,17 @@ function getStoppingStrings(isImpersonate) {
  * @param {string} quiet_prompt Instruction prompt for the AI
  * @param {boolean} quietToLoud Whether the message should be sent in a foreground (loud) or background (quiet) mode
  * @param {boolean} skipWIAN whether to skip addition of World Info and Author's Note into the prompt
+ * @param {string} quietImage Image to use for the quiet prompt
  * @returns
  */
-export async function generateQuietPrompt(quiet_prompt, quietToLoud, skipWIAN) {
+export async function generateQuietPrompt(quiet_prompt, quietToLoud, skipWIAN, quietImage = null) {
     console.log('got into genQuietPrompt')
     const skipWIANvalue = skipWIAN
     return await new Promise(
         async function promptPromise(resolve, reject) {
             if (quietToLoud === true) {
                 try {
-                    await Generate('quiet', { resolve, reject, quiet_prompt, quietToLoud: true, skipWIAN: skipWIAN, force_name2: true, });
+                    await Generate('quiet', { resolve, reject, quiet_prompt, quietToLoud: true, skipWIAN: skipWIAN, force_name2: true, quietImage: quietImage });
                 }
                 catch {
                     reject();
@@ -2122,7 +2123,7 @@ export async function generateQuietPrompt(quiet_prompt, quietToLoud, skipWIAN) {
             else {
                 try {
                     console.log('going to generate non-QuietToLoud')
-                    await Generate('quiet', { resolve, reject, quiet_prompt, quietToLoud: false, skipWIAN: skipWIAN, force_name2: true, });
+                    await Generate('quiet', { resolve, reject, quiet_prompt, quietToLoud: false, skipWIAN: skipWIAN, force_name2: true, quietImage: quietImage });
                 }
                 catch {
                     reject();
@@ -2671,7 +2672,7 @@ export async function generateRaw(prompt, api) {
     return message;
 }
 
-async function Generate(type, { automatic_trigger, force_name2, resolve, reject, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal } = {}, dryRun = false) {
+async function Generate(type, { automatic_trigger, force_name2, resolve, reject, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage } = {}, dryRun = false) {
     console.log('Generate entered');
     setGenerationProgress(0);
     generation_started = new Date();
@@ -2726,7 +2727,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
     }
 
     if (selected_group && !is_group_generating && !dryRun) {
-        generateGroupWrapper(false, type, { resolve, reject, quiet_prompt, force_chid, signal: abortController.signal });
+        generateGroupWrapper(false, type, { resolve, reject, quiet_prompt, force_chid, signal: abortController.signal, quietImage });
         return;
     } else if (selected_group && !is_group_generating && dryRun) {
         const characterIndexMap = new Map(characters.map((char, index) => [char.avatar, index]));
@@ -3432,6 +3433,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
                     bias: promptBias,
                     type: type,
                     quietPrompt: quiet_prompt,
+                    quietImage: quietImage,
                     cyclePrompt: cyclePrompt,
                     systemPromptOverride: system,
                     jailbreakPromptOverride: jailbreak,
@@ -6812,6 +6814,7 @@ window["SillyTavern"].getContext = function () {
         extensionSettings: extension_settings,
         ModuleWorkerWrapper: ModuleWorkerWrapper,
         getTokenizerModel: getTokenizerModel,
+        generateQuietPrompt: generateQuietPrompt,
         tags: tags,
         tagMap: tag_map,
     };

public/scripts/extensions.js

@@ -103,7 +103,7 @@ class ModuleWorkerWrapper {
     }
 
     // Called by the extension
-    async update() {
+    async update(...args) {
         // Don't touch me I'm busy...
         if (this.isBusy) {
             return;
@@ -112,7 +112,7 @@ class ModuleWorkerWrapper {
         // I'm free. Let's update!
         try {
             this.isBusy = true;
-            await this.callback();
+            await this.callback(...args);
         }
         finally {
             this.isBusy = false;

public/scripts/extensions/tts/index.js

@@ -303,6 +303,7 @@ window.debugTtsPlayback = debugTtsPlayback
 //##################//
 
 let audioElement = new Audio()
+audioElement.id = 'tts_audio'
 audioElement.autoplay = true
 
 let audioJobQueue = []
@@ -989,4 +990,5 @@ $(document).ready(function () {
     eventSource.on(event_types.MESSAGE_DELETED, onChatDeleted);
     eventSource.on(event_types.GROUP_UPDATED, onChatChanged)
     registerSlashCommand('speak', onNarrateText, ['narrate', 'tts'], `<span class="monospace">(text)</span>  – narrate any text using currently selected character's voice. Use voice="Character Name" argument to set other voice from the voice map, example: <tt>/speak voice="Donald Duck" Quack!</tt>`, true, true);
+    document.body.appendChild(audioElement);
 })

public/scripts/openai.js

@@ -742,9 +742,10 @@ function getPromptPosition(position) {
  * @param {Object} options - An object with optional settings.
  * @param {string} options.bias - A bias to be added in the conversation.
  * @param {string} options.quietPrompt - Instruction prompt for extras
+ * @param {string} options.quietImage - Image prompt for extras
  * @param {string} options.type - The type of the chat, can be 'impersonate'.
  */
-async function populateChatCompletion(prompts, chatCompletion, { bias, quietPrompt, type, cyclePrompt } = {}) {
+async function populateChatCompletion(prompts, chatCompletion, { bias, quietPrompt, quietImage, type, cyclePrompt } = {}) {
     // Helper function for preparing a prompt, that already exists within the prompt collection, for completion
     const addToChatCompletion = (source, target = null) => {
         // We need the prompts array to determine a position for the source.
@@ -781,7 +782,13 @@ async function populateChatCompletion(prompts, chatCompletion, { bias, quietProm
     // Add quiet prompt to control prompts
     // This should always be last, even in control prompts. Add all further control prompts BEFORE this prompt
     const quietPromptMessage = Message.fromPrompt(prompts.get('quietPrompt')) ?? null;
-    if (quietPromptMessage && quietPromptMessage.content) controlPrompts.add(quietPromptMessage);
+    if (quietPromptMessage && quietPromptMessage.content) {
+        if (isImageInliningSupported() && quietImage) {
+            await quietPromptMessage.addImage(quietImage);
+        }
+
+        controlPrompts.add(quietPromptMessage);
+    }
 
     chatCompletion.reserveBudget(controlPrompts);
 
@@ -1004,6 +1011,7 @@ export async function prepareOpenAIMessages({
     bias,
     type,
     quietPrompt,
+    quietImage,
     extensionPrompts,
     cyclePrompt,
     systemPromptOverride,
@@ -1029,6 +1037,7 @@ export async function prepareOpenAIMessages({
             worldInfoAfter,
             charDescription,
             quietPrompt,
+            quietImage,
             bias,
             extensionPrompts,
             systemPromptOverride,
@@ -1037,7 +1046,7 @@ export async function prepareOpenAIMessages({
         });
 
         // Fill the chat completion with as much context as the budget allows
-        await populateChatCompletion(prompts, chatCompletion, { bias, quietPrompt, type, cyclePrompt });
+        await populateChatCompletion(prompts, chatCompletion, { bias, quietPrompt, quietImage, type, cyclePrompt });
     } catch (error) {
         if (error instanceof TokenBudgetExceededError) {
             toastr.error('An error occurred while counting tokens: Token budget exceeded.')

src/openai.js

@@ -27,7 +27,7 @@ function registerEndpoints(app, jsonParser) {
                         ]
                     }
                 ],
-                max_tokens: 300
+                max_tokens: 500
             };
 
             console.log('OpenAI request', body);