From 6efa36bdbf2d6f17530a94fd36a3cb825fc1a865 Mon Sep 17 00:00:00 2001 From: Honey Tree Date: Thu, 10 Oct 2024 09:52:54 -0300 Subject: [PATCH 01/10] Expand WORLD_INFO_FORCE_ACTIVATE API to allow for dynamically modified Lorebook entries --- public/scripts/world-info.js | 58 ++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index 6aad5d92e..7b9c3736c 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -130,9 +130,9 @@ const KNOWN_DECORATORS = ['@@activate', '@@dont_activate']; */ class WorldInfoBuffer { /** - * @type {object[]} Array of entries that need to be activated no matter what + * @type {Map} Map of entries that need to be activated no matter what */ - static externalActivations = []; + static externalActivations = new Map(); /** * @type {string[]} Array of messages sorted by ascending depth @@ -311,20 +311,20 @@ class WorldInfoBuffer { } /** - * Check if the current entry is externally activated. + * Get the externally activated version of the entry, if there is one. * @param {object} entry WI entry to check - * @returns {boolean} True if the entry is forcefully activated + * @returns {object|null} the external version if the entry is forcefully activated, null otherwise */ - isExternallyActivated(entry) { + getExternallyActivated(entry) { // Entries could be copied with structuredClone, so we need to compare them by string representation - return WorldInfoBuffer.externalActivations.some(x => JSON.stringify(x) === JSON.stringify(entry)); + return WorldInfoBuffer.externalActivations.get(`${entry.world}.${entry.uid}`); } /** * Clean-up the external effects for entries. */ resetExternalEffects() { - WorldInfoBuffer.externalActivations.splice(0, WorldInfoBuffer.externalActivations.length); + WorldInfoBuffer.externalActivations = new Map(); } /** @@ -751,7 +751,7 @@ export async function getWorldInfoPrompt(chat, maxContext, isDryRun) { worldInfoString = worldInfoBefore + worldInfoAfter; if (!isDryRun && activatedWorldInfo.allActivatedEntries && activatedWorldInfo.allActivatedEntries.size > 0) { - const arg = Array.from(activatedWorldInfo.allActivatedEntries); + const arg = Array.from(activatedWorldInfo.allActivatedEntries.values()); await eventSource.emit(event_types.WORLD_INFO_ACTIVATED, arg); } @@ -868,7 +868,13 @@ export function setWorldInfoSettings(settings, data) { }); eventSource.on(event_types.WORLDINFO_FORCE_ACTIVATE, (entries) => { - WorldInfoBuffer.externalActivations.push(...entries); + for (const entry of entries) { + if (!Object.hasOwn(entry, 'world') || !Object.hasOwn(entry, 'uid')) { + console.error('WORLDINFO_FORCE_ACTIVATE requires all entries to have both world and uid fields, entry IGNORED'); + } else { + WorldInfoBuffer.externalActivations.set(`${entry.world}.${entry.uid}`, entry); + } + } }); // Add slash commands @@ -3698,7 +3704,7 @@ function parseDecorators(content) { * @param {string[]} chat The chat messages to scan, in reverse order. * @param {number} maxContext The maximum context size of the generation. * @param {boolean} isDryRun Whether to perform a dry run. - * @typedef {{ worldInfoBefore: string, worldInfoAfter: string, EMEntries: any[], WIDepthEntries: any[], allActivatedEntries: Set }} WIActivated + * @typedef {{ worldInfoBefore: string, worldInfoAfter: string, EMEntries: any[], WIDepthEntries: any[], allActivatedEntries: Map }} WIActivated * @returns {Promise} The world info activated. */ export async function checkWorldInfo(chat, maxContext, isDryRun) { @@ -3724,7 +3730,7 @@ export async function checkWorldInfo(chat, maxContext, isDryRun) { let scanState = scan_state.INITIAL; let token_budget_overflowed = false; let count = 0; - let allActivatedEntries = new Set(); + let allActivatedEntries = new Map(); let failedProbabilityChecks = new Set(); let allActivatedText = ''; @@ -3742,7 +3748,7 @@ export async function checkWorldInfo(chat, maxContext, isDryRun) { !isDryRun && timedEffects.checkTimedEffects(); if (sortedEntries.length === 0) { - return { worldInfoBefore: '', worldInfoAfter: '', WIDepthEntries: [], EMEntries: [], allActivatedEntries: new Set() }; + return { worldInfoBefore: '', worldInfoAfter: '', WIDepthEntries: [], EMEntries: [], allActivatedEntries: new Map() }; } /** @type {number[]} Represents the delay levels for entries that are delayed until recursion */ @@ -3789,7 +3795,7 @@ export async function checkWorldInfo(chat, maxContext, isDryRun) { } // Already processed, considered and then skipped entries should still be skipped - if (failedProbabilityChecks.has(entry) || allActivatedEntries.has(entry)) { + if (failedProbabilityChecks.has(entry) || allActivatedEntries.has(`${entry.world}.${entry.uid}`)) { continue; } @@ -3858,6 +3864,12 @@ export async function checkWorldInfo(chat, maxContext, isDryRun) { continue; } + if (buffer.getExternallyActivated(entry)) { + log('externally activated'); + activatedNow.add(buffer.getExternallyActivated(entry)); + continue; + } + if (entry.decorators.includes('@@activate')) { log('activated by @@activate decorator'); activatedNow.add(entry); @@ -3876,12 +3888,6 @@ export async function checkWorldInfo(chat, maxContext, isDryRun) { continue; } - if (buffer.isExternallyActivated(entry)) { - log('externally activated'); - activatedNow.add(entry); - continue; - } - if (isSticky) { log('activated because active sticky'); activatedNow.add(entry); @@ -4039,7 +4045,7 @@ export async function checkWorldInfo(chat, maxContext, isDryRun) { break; } - allActivatedEntries.add(entry); + allActivatedEntries.set(`${entry.world}.${entry.uid}`, entry); console.debug(`[WI] Entry ${entry.uid} activation successful, adding to prompt`, entry); } @@ -4123,7 +4129,7 @@ export async function checkWorldInfo(chat, maxContext, isDryRun) { // Appends from insertion order 999 to 1. Use unshift for this purpose // TODO (kingbri): Change to use WI Anchor positioning instead of separate top/bottom arrays - [...allActivatedEntries].sort(sortFn).forEach((entry) => { + [...allActivatedEntries.values()].sort(sortFn).forEach((entry) => { const regexDepth = entry.position === world_info_position.atDepth ? (entry.depth ?? DEFAULT_DEPTH) : null; const content = getRegexedString(entry.content, regex_placement.WORLD_INFO, { depth: regexDepth, isMarkdown: false, isPrompt: true }); @@ -4182,14 +4188,14 @@ export async function checkWorldInfo(chat, maxContext, isDryRun) { context.setExtensionPrompt(NOTE_MODULE_NAME, ANWithWI, chat_metadata[metadata_keys.position], chat_metadata[metadata_keys.depth], extension_settings.note.allowWIScan, chat_metadata[metadata_keys.role]); } - !isDryRun && timedEffects.setTimedEffects(Array.from(allActivatedEntries)); + !isDryRun && timedEffects.setTimedEffects(Array.from(allActivatedEntries.values())); buffer.resetExternalEffects(); timedEffects.cleanUp(); - console.log(`[WI] Adding ${allActivatedEntries.size} entries to prompt`, Array.from(allActivatedEntries)); + console.log(`[WI] Adding ${allActivatedEntries.size} entries to prompt`, Array.from(allActivatedEntries.values())); console.debug('[WI] --- DONE ---'); - return { worldInfoBefore, worldInfoAfter, EMEntries, WIDepthEntries, allActivatedEntries }; + return { worldInfoBefore, worldInfoAfter, EMEntries, WIDepthEntries, allActivatedEntries: allActivatedEntries.values() }; } /** @@ -4291,7 +4297,7 @@ function filterGroupsByTimedEffects(groups, timedEffects, removeEntry) { /** * Filters entries by inclusion groups. * @param {object[]} newEntries Entries activated on current recursion level - * @param {Set} allActivatedEntries Set of all activated entries + * @param {Map} allActivatedEntries Map of all activated entries * @param {WorldInfoBuffer} buffer The buffer to use for scanning * @param {number} scanState The current scan state * @param {WorldInfoTimedEffects} timedEffects The timed effects currently active @@ -4339,7 +4345,7 @@ function filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanSt continue; } - if (Array.from(allActivatedEntries).some(x => x.group === key)) { + if (Array.from(allActivatedEntries.values()).some(x => x.group === key)) { console.debug(`[WI] Skipping inclusion group check, group '${key}' was already activated`); // We need to forcefully deactivate all other entries in the group removeAllBut(group, null, false); From 7fd798b854fe142c04dd08f5ec6a08c8bc568e0b Mon Sep 17 00:00:00 2001 From: Honey Tree Date: Thu, 10 Oct 2024 10:12:19 -0300 Subject: [PATCH 02/10] Typing fixes to avoid breaking any APIs --- public/scripts/world-info.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index 7b9c3736c..5c7a952f4 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -3704,7 +3704,7 @@ function parseDecorators(content) { * @param {string[]} chat The chat messages to scan, in reverse order. * @param {number} maxContext The maximum context size of the generation. * @param {boolean} isDryRun Whether to perform a dry run. - * @typedef {{ worldInfoBefore: string, worldInfoAfter: string, EMEntries: any[], WIDepthEntries: any[], allActivatedEntries: Map }} WIActivated + * @typedef {{ worldInfoBefore: string, worldInfoAfter: string, EMEntries: any[], WIDepthEntries: any[], allActivatedEntries: Set }} WIActivated * @returns {Promise} The world info activated. */ export async function checkWorldInfo(chat, maxContext, isDryRun) { @@ -3748,7 +3748,7 @@ export async function checkWorldInfo(chat, maxContext, isDryRun) { !isDryRun && timedEffects.checkTimedEffects(); if (sortedEntries.length === 0) { - return { worldInfoBefore: '', worldInfoAfter: '', WIDepthEntries: [], EMEntries: [], allActivatedEntries: new Map() }; + return { worldInfoBefore: '', worldInfoAfter: '', WIDepthEntries: [], EMEntries: [], allActivatedEntries: new Set() }; } /** @type {number[]} Represents the delay levels for entries that are delayed until recursion */ @@ -4195,7 +4195,7 @@ export async function checkWorldInfo(chat, maxContext, isDryRun) { console.log(`[WI] Adding ${allActivatedEntries.size} entries to prompt`, Array.from(allActivatedEntries.values())); console.debug('[WI] --- DONE ---'); - return { worldInfoBefore, worldInfoAfter, EMEntries, WIDepthEntries, allActivatedEntries: allActivatedEntries.values() }; + return { worldInfoBefore, worldInfoAfter, EMEntries, WIDepthEntries, allActivatedEntries: new Set(allActivatedEntries.values()) }; } /** From 0d1e96cc174a2ab063727fbb3734c309b78de57a Mon Sep 17 00:00:00 2001 From: Honey Tree Date: Thu, 10 Oct 2024 11:44:13 -0300 Subject: [PATCH 03/10] Map.get returns undefined and not null on empty --- public/scripts/world-info.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index 5c7a952f4..ccc5f0d4e 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -313,7 +313,7 @@ class WorldInfoBuffer { /** * Get the externally activated version of the entry, if there is one. * @param {object} entry WI entry to check - * @returns {object|null} the external version if the entry is forcefully activated, null otherwise + * @returns {object|undefined} the external version if the entry is forcefully activated, undefined otherwise */ getExternallyActivated(entry) { // Entries could be copied with structuredClone, so we need to compare them by string representation From 61ba579fe2223a95db556105e20d95aa8b081e3c Mon Sep 17 00:00:00 2001 From: Honey Tree Date: Thu, 10 Oct 2024 12:03:26 -0300 Subject: [PATCH 04/10] Moving check down for spec adherence --- public/scripts/world-info.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index ccc5f0d4e..924a0e902 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -3864,12 +3864,6 @@ export async function checkWorldInfo(chat, maxContext, isDryRun) { continue; } - if (buffer.getExternallyActivated(entry)) { - log('externally activated'); - activatedNow.add(buffer.getExternallyActivated(entry)); - continue; - } - if (entry.decorators.includes('@@activate')) { log('activated by @@activate decorator'); activatedNow.add(entry); @@ -3881,6 +3875,12 @@ export async function checkWorldInfo(chat, maxContext, isDryRun) { continue; } + if (buffer.getExternallyActivated(entry)) { + log('externally activated'); + activatedNow.add(buffer.getExternallyActivated(entry)); + continue; + } + // Now do checks for immediate activations if (entry.constant) { log('activated because of constant'); From fe33519422c8a62cd73aad33ec5473fe07314597 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:32:45 +0000 Subject: [PATCH 05/10] Image Generation: Don't replace dots with commas --- public/scripts/extensions/stable-diffusion/index.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js index e9d3a7632..8e96370c3 100644 --- a/public/scripts/extensions/stable-diffusion/index.js +++ b/public/scripts/extensions/stable-diffusion/index.js @@ -2208,10 +2208,9 @@ function processReply(str) { str = str.replaceAll('"', ''); str = str.replaceAll('“', ''); - str = str.replaceAll('.', ','); str = str.replaceAll('\n', ', '); str = str.normalize('NFD'); - str = str.replace(/[^a-zA-Z0-9,:_(){}<>[\]\-']+/g, ' '); + str = str.replace(/[^a-zA-Z0-9\.,:_(){}<>[\]\-']+/g, ' '); str = str.replace(/\s+/g, ' '); // Collapse multiple whitespaces into one str = str.trim(); From ed8e379e542e926a5f823abb6b1723f2197a889f Mon Sep 17 00:00:00 2001 From: Devin Bayer Date: Fri, 11 Oct 2024 17:48:33 +0200 Subject: [PATCH 06/10] support new together.ai image generation api --- .../extensions/stable-diffusion/index.js | 3 +-- src/endpoints/stable-diffusion.js | 18 +++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js index e9d3a7632..58e69f4f2 100644 --- a/public/scripts/extensions/stable-diffusion/index.js +++ b/public/scripts/extensions/stable-diffusion/index.js @@ -2675,8 +2675,7 @@ async function generateTogetherAIImage(prompt, negativePrompt, signal) { }); if (result.ok) { - const data = await result.json(); - return { format: 'jpg', data: data?.output?.choices?.[0]?.image_base64 }; + return await result.json(); } else { const text = await result.text(); throw new Error(text); diff --git a/src/endpoints/stable-diffusion.js b/src/endpoints/stable-diffusion.js index b6348b8b2..0ac395a4d 100644 --- a/src/endpoints/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -607,10 +607,9 @@ together.post('/generate', jsonParser, async (request, response) => { console.log('TogetherAI request:', request.body); - const result = await fetch('https://api.together.xyz/api/inference', { + const result = await fetch('https://api.together.xyz/v1/images/generations', { method: 'POST', body: JSON.stringify({ - request_type: 'image-model-inference', prompt: request.body.prompt, negative_prompt: request.body.negative_prompt, height: request.body.height, @@ -620,8 +619,6 @@ together.post('/generate', jsonParser, async (request, response) => { n: 1, // Limited to 10000 on playground, works fine with more. seed: request.body.seed >= 0 ? request.body.seed : Math.floor(Math.random() * 10_000_000), - // Don't know if that's supposed to be random or not. It works either way. - sessionKey: getHexString(40), }), headers: { 'Content-Type': 'application/json', @@ -630,19 +627,22 @@ together.post('/generate', jsonParser, async (request, response) => { }); if (!result.ok) { - console.log('TogetherAI returned an error.'); + console.log('TogetherAI returned an error.', { body: await result.text() }); return response.sendStatus(500); } const data = await result.json(); console.log('TogetherAI response:', data); - if (data.status !== 'finished') { - console.log('TogetherAI job failed.'); - return response.sendStatus(500); + const choice = data?.data?.[0]; + let b64_json = choice.b64_json; + + if(! b64_json) { + const buffer = await (await fetch(choice.url)).buffer(); + b64_json = buffer.toString('base64'); } - return response.send(data); + return response.send({ format: 'jpg', data: b64_json }); } catch (error) { console.log(error); return response.sendStatus(500); From 34b1fa2cffbef30e10b122bcc0502895b24893e1 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:28:54 +0300 Subject: [PATCH 07/10] [chore] Run code format --- src/endpoints/stable-diffusion.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/endpoints/stable-diffusion.js b/src/endpoints/stable-diffusion.js index 0ac395a4d..986774d93 100644 --- a/src/endpoints/stable-diffusion.js +++ b/src/endpoints/stable-diffusion.js @@ -637,7 +637,7 @@ together.post('/generate', jsonParser, async (request, response) => { const choice = data?.data?.[0]; let b64_json = choice.b64_json; - if(! b64_json) { + if (!b64_json) { const buffer = await (await fetch(choice.url)).buffer(); b64_json = buffer.toString('base64'); } From 161fc7f0f091395f7ec34a9ace91596651363ad5 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 21:01:32 +0300 Subject: [PATCH 08/10] STscript: don't parse boolean operands as 0 for strings containing just spaces --- public/scripts/variables.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/public/scripts/variables.js b/public/scripts/variables.js index f2c340541..7fc12aa54 100644 --- a/public/scripts/variables.js +++ b/public/scripts/variables.js @@ -514,7 +514,8 @@ export function parseBooleanOperands(args) { return ''; } - const operandNumber = Number(operand); + // parseFloat will return NaN for spaces. + const operandNumber = parseFloat(operand); if (!isNaN(operandNumber)) { return operandNumber; From b65da795ef0d75d5841b2dbfaf24fd8f07988ba6 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 21:59:55 +0300 Subject: [PATCH 09/10] [chore] Fix comments/logs --- public/scripts/world-info.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index 508f348fe..ef0ae1c4c 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -316,7 +316,6 @@ class WorldInfoBuffer { * @returns {object|undefined} the external version if the entry is forcefully activated, undefined otherwise */ getExternallyActivated(entry) { - // Entries could be copied with structuredClone, so we need to compare them by string representation return WorldInfoBuffer.externalActivations.get(`${entry.world}.${entry.uid}`); } @@ -870,7 +869,7 @@ export function setWorldInfoSettings(settings, data) { eventSource.on(event_types.WORLDINFO_FORCE_ACTIVATE, (entries) => { for (const entry of entries) { if (!Object.hasOwn(entry, 'world') || !Object.hasOwn(entry, 'uid')) { - console.error('WORLDINFO_FORCE_ACTIVATE requires all entries to have both world and uid fields, entry IGNORED'); + console.error('[WI] WORLDINFO_FORCE_ACTIVATE requires all entries to have both world and uid fields, entry IGNORED', entry); } else { WorldInfoBuffer.externalActivations.set(`${entry.world}.${entry.uid}`, entry); } From 6f4e98024a25dec1dd2d731bc2b0c9dfce113f46 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Fri, 11 Oct 2024 22:10:25 +0300 Subject: [PATCH 10/10] [chore] Add a log on force activation --- public/scripts/world-info.js | 1 + 1 file changed, 1 insertion(+) diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index ef0ae1c4c..05e6c6630 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -872,6 +872,7 @@ export function setWorldInfoSettings(settings, data) { console.error('[WI] WORLDINFO_FORCE_ACTIVATE requires all entries to have both world and uid fields, entry IGNORED', entry); } else { WorldInfoBuffer.externalActivations.set(`${entry.world}.${entry.uid}`, entry); + console.log('[WI] WORLDINFO_FORCE_ACTIVATE added entry', entry); } } });