From 323b338cdd08cd3abb5be7d757c5c5b85d8bb869 Mon Sep 17 00:00:00 2001
From: Cohee <18619528+Cohee1207@users.noreply.github.com>
Date: Fri, 17 Nov 2023 01:30:32 +0200
Subject: [PATCH] Add images to quiet prompts if inlining enabled
---
public/script.js | 13 ++++++++-----
public/scripts/extensions.js | 4 ++--
public/scripts/extensions/tts/index.js | 2 ++
public/scripts/openai.js | 15 ++++++++++++---
src/openai.js | 2 +-
5 files changed, 25 insertions(+), 11 deletions(-)
diff --git a/public/script.js b/public/script.js
index 600dfa0ce..9ff299d9c 100644
--- a/public/script.js
+++ b/public/script.js
@@ -2104,16 +2104,17 @@ function getStoppingStrings(isImpersonate) {
* @param {string} quiet_prompt Instruction prompt for the AI
* @param {boolean} quietToLoud Whether the message should be sent in a foreground (loud) or background (quiet) mode
* @param {boolean} skipWIAN whether to skip addition of World Info and Author's Note into the prompt
+ * @param {string} quietImage Image to use for the quiet prompt
* @returns
*/
-export async function generateQuietPrompt(quiet_prompt, quietToLoud, skipWIAN) {
+export async function generateQuietPrompt(quiet_prompt, quietToLoud, skipWIAN, quietImage = null) {
console.log('got into genQuietPrompt')
const skipWIANvalue = skipWIAN
return await new Promise(
async function promptPromise(resolve, reject) {
if (quietToLoud === true) {
try {
- await Generate('quiet', { resolve, reject, quiet_prompt, quietToLoud: true, skipWIAN: skipWIAN, force_name2: true, });
+ await Generate('quiet', { resolve, reject, quiet_prompt, quietToLoud: true, skipWIAN: skipWIAN, force_name2: true, quietImage: quietImage });
}
catch {
reject();
@@ -2122,7 +2123,7 @@ export async function generateQuietPrompt(quiet_prompt, quietToLoud, skipWIAN) {
else {
try {
console.log('going to generate non-QuietToLoud')
- await Generate('quiet', { resolve, reject, quiet_prompt, quietToLoud: false, skipWIAN: skipWIAN, force_name2: true, });
+ await Generate('quiet', { resolve, reject, quiet_prompt, quietToLoud: false, skipWIAN: skipWIAN, force_name2: true, quietImage: quietImage });
}
catch {
reject();
@@ -2671,7 +2672,7 @@ export async function generateRaw(prompt, api) {
return message;
}
-async function Generate(type, { automatic_trigger, force_name2, resolve, reject, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal } = {}, dryRun = false) {
+async function Generate(type, { automatic_trigger, force_name2, resolve, reject, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage } = {}, dryRun = false) {
console.log('Generate entered');
setGenerationProgress(0);
generation_started = new Date();
@@ -2726,7 +2727,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
}
if (selected_group && !is_group_generating && !dryRun) {
- generateGroupWrapper(false, type, { resolve, reject, quiet_prompt, force_chid, signal: abortController.signal });
+ generateGroupWrapper(false, type, { resolve, reject, quiet_prompt, force_chid, signal: abortController.signal, quietImage });
return;
} else if (selected_group && !is_group_generating && dryRun) {
const characterIndexMap = new Map(characters.map((char, index) => [char.avatar, index]));
@@ -3432,6 +3433,7 @@ async function Generate(type, { automatic_trigger, force_name2, resolve, reject,
bias: promptBias,
type: type,
quietPrompt: quiet_prompt,
+ quietImage: quietImage,
cyclePrompt: cyclePrompt,
systemPromptOverride: system,
jailbreakPromptOverride: jailbreak,
@@ -6812,6 +6814,7 @@ window["SillyTavern"].getContext = function () {
extensionSettings: extension_settings,
ModuleWorkerWrapper: ModuleWorkerWrapper,
getTokenizerModel: getTokenizerModel,
+ generateQuietPrompt: generateQuietPrompt,
tags: tags,
tagMap: tag_map,
};
diff --git a/public/scripts/extensions.js b/public/scripts/extensions.js
index b95228af0..254175bb2 100644
--- a/public/scripts/extensions.js
+++ b/public/scripts/extensions.js
@@ -103,7 +103,7 @@ class ModuleWorkerWrapper {
}
// Called by the extension
- async update() {
+ async update(...args) {
// Don't touch me I'm busy...
if (this.isBusy) {
return;
@@ -112,7 +112,7 @@ class ModuleWorkerWrapper {
// I'm free. Let's update!
try {
this.isBusy = true;
- await this.callback();
+ await this.callback(...args);
}
finally {
this.isBusy = false;
diff --git a/public/scripts/extensions/tts/index.js b/public/scripts/extensions/tts/index.js
index 2ff01190a..7e8c1cd4e 100644
--- a/public/scripts/extensions/tts/index.js
+++ b/public/scripts/extensions/tts/index.js
@@ -303,6 +303,7 @@ window.debugTtsPlayback = debugTtsPlayback
//##################//
let audioElement = new Audio()
+audioElement.id = 'tts_audio'
audioElement.autoplay = true
let audioJobQueue = []
@@ -989,4 +990,5 @@ $(document).ready(function () {
eventSource.on(event_types.MESSAGE_DELETED, onChatDeleted);
eventSource.on(event_types.GROUP_UPDATED, onChatChanged)
registerSlashCommand('speak', onNarrateText, ['narrate', 'tts'], `(text) – narrate any text using currently selected character's voice. Use voice="Character Name" argument to set other voice from the voice map, example: /speak voice="Donald Duck" Quack!`, true, true);
+ document.body.appendChild(audioElement);
})
diff --git a/public/scripts/openai.js b/public/scripts/openai.js
index 70c3fe304..b661be7ae 100644
--- a/public/scripts/openai.js
+++ b/public/scripts/openai.js
@@ -742,9 +742,10 @@ function getPromptPosition(position) {
* @param {Object} options - An object with optional settings.
* @param {string} options.bias - A bias to be added in the conversation.
* @param {string} options.quietPrompt - Instruction prompt for extras
+ * @param {string} options.quietImage - Image prompt for extras
* @param {string} options.type - The type of the chat, can be 'impersonate'.
*/
-async function populateChatCompletion(prompts, chatCompletion, { bias, quietPrompt, type, cyclePrompt } = {}) {
+async function populateChatCompletion(prompts, chatCompletion, { bias, quietPrompt, quietImage, type, cyclePrompt } = {}) {
// Helper function for preparing a prompt, that already exists within the prompt collection, for completion
const addToChatCompletion = (source, target = null) => {
// We need the prompts array to determine a position for the source.
@@ -781,7 +782,13 @@ async function populateChatCompletion(prompts, chatCompletion, { bias, quietProm
// Add quiet prompt to control prompts
// This should always be last, even in control prompts. Add all further control prompts BEFORE this prompt
const quietPromptMessage = Message.fromPrompt(prompts.get('quietPrompt')) ?? null;
- if (quietPromptMessage && quietPromptMessage.content) controlPrompts.add(quietPromptMessage);
+ if (quietPromptMessage && quietPromptMessage.content) {
+ if (isImageInliningSupported() && quietImage) {
+ await quietPromptMessage.addImage(quietImage);
+ }
+
+ controlPrompts.add(quietPromptMessage);
+ }
chatCompletion.reserveBudget(controlPrompts);
@@ -1004,6 +1011,7 @@ export async function prepareOpenAIMessages({
bias,
type,
quietPrompt,
+ quietImage,
extensionPrompts,
cyclePrompt,
systemPromptOverride,
@@ -1029,6 +1037,7 @@ export async function prepareOpenAIMessages({
worldInfoAfter,
charDescription,
quietPrompt,
+ quietImage,
bias,
extensionPrompts,
systemPromptOverride,
@@ -1037,7 +1046,7 @@ export async function prepareOpenAIMessages({
});
// Fill the chat completion with as much context as the budget allows
- await populateChatCompletion(prompts, chatCompletion, { bias, quietPrompt, type, cyclePrompt });
+ await populateChatCompletion(prompts, chatCompletion, { bias, quietPrompt, quietImage, type, cyclePrompt });
} catch (error) {
if (error instanceof TokenBudgetExceededError) {
toastr.error('An error occurred while counting tokens: Token budget exceeded.')
diff --git a/src/openai.js b/src/openai.js
index 857c0198f..1516715d4 100644
--- a/src/openai.js
+++ b/src/openai.js
@@ -27,7 +27,7 @@ function registerEndpoints(app, jsonParser) {
]
}
],
- max_tokens: 300
+ max_tokens: 500
};
console.log('OpenAI request', body);