rixty
/
SillyTavern
mirror of https://github.com/SillyTavern/SillyTavern.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2479 from SillyTavern/wi-processing-refactoring 

WI scan refactoring:  Extended logging, updated code flow, changed Min Activations
This commit is contained in:
Cohee 2024-07-14 22:53:05 +03:00 committed by GitHub
parent 6ba384f895 d2ca90bb2c
commit 9268fc3ef2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 263 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

423
public/scripts/world-info.js
View File

@ -169,7 +169,7 @@ class WorldInfoBuffer {
#skew = 0;
/**
* @type {number} The starting depth of the global scan depth. Incremented by "min activations" feature to not repeat scans. When > 0 it means a complete scan was done up to #startDepth already, and `advanceScanPosition` was called.
* @type {number} The starting depth of the global scan depth.
*/
#startDepth = 0;
@ -222,12 +222,12 @@ class WorldInfoBuffer {
}
if (depth < 0) {
console.error(`Invalid WI scan depth ${depth}. Must be >= 0`);
console.error(`[WI] Invalid WI scan depth ${depth}. Must be >= 0`);
return '';
}
if (depth > MAX_SCAN_DEPTH) {
console.warn(`Invalid WI scan depth ${depth}. Truncating to ${MAX_SCAN_DEPTH}`);
console.warn(`[WI] Invalid WI scan depth ${depth}. Truncating to ${MAX_SCAN_DEPTH}`);
depth = MAX_SCAN_DEPTH;
}
@ -301,10 +301,17 @@ class WorldInfoBuffer {
}
/**
* Increments skew and sets startDepth to previous depth.
* Checks if the recursion buffer is not empty.
* @returns {boolean} Returns true if the recursion buffer is not empty, otherwise false
*/
advanceScanPosition() {
this.#startDepth = this.getDepth();
hasRecurse() {
return this.#recurseBuffer.length > 0;
}
/**
* Increments skew to advance the scan range.
*/
advanceScan() {
this.#skew++;
}
@ -436,7 +443,7 @@ class WorldInfoTimedEffects {
const key = this.#getEntryKey(entry);
const effect = this.#getEntryTimedEffect('cooldown', entry, true);
chat_metadata.timedWorldInfo.cooldown[key] = effect;
console.log(`Adding cooldown entry ${key} on ended sticky: start=${effect.start}, end=${effect.end}, protected=${effect.protected}`);
console.log(`[WI] Adding cooldown entry ${key} on ended sticky: start=${effect.start}, end=${effect.end}, protected=${effect.protected}`);
// Set the cooldown immediately for this evaluation
this.#buffer.cooldown.push(entry);
},
@ -447,10 +454,10 @@ class WorldInfoTimedEffects {
* @param {WIScanEntry} entry Entry that ended cooldown
*/
'cooldown': (entry) => {
console.debug('Cooldown ended for entry', entry.uid);
console.debug('[WI] Cooldown ended for entry', entry.uid);
},
'delay': () => {},
'delay': () => { },
};
/**
@ -537,11 +544,11 @@ class WorldInfoTimedEffects {
/** @type {[string, WITimedEffect][]} */
const effects = Object.entries(chat_metadata.timedWorldInfo[type]);
for (const [key, value] of effects) {
console.log(`Processing ${type} entry ${key}`, value);
console.log(`[WI] Processing ${type} entry ${key}`, value);
const entry = this.#entries.find(x => String(this.#getEntryHash(x)) === String(value.hash));
if (this.#chat.length <= Number(value.start) && !value.protected) {
console.log(`Removing ${type} entry ${key} from timedWorldInfo: chat not advanced`, value);
console.log(`[WI] Removing ${type} entry ${key} from timedWorldInfo: chat not advanced`, value);
delete chat_metadata.timedWorldInfo[type][key];
continue;
}
@ -549,7 +556,7 @@ class WorldInfoTimedEffects {
// Missing entries (they could be from another character's lorebook)
if (!entry) {
if (this.#chat.length >= Number(value.end)) {
console.log(`Removing ${type} entry from timedWorldInfo: entry not found and interval passed`, entry);
console.log(`[WI] Removing ${type} entry from timedWorldInfo: entry not found and interval passed`, entry);
delete chat_metadata.timedWorldInfo[type][key];
}
continue;
@ -557,13 +564,13 @@ class WorldInfoTimedEffects {
// Ignore invalid entries (not configured for timed effects)
if (!entry[type]) {
console.log(`Removing ${type} entry from timedWorldInfo: entry not ${type}`, entry);
console.log(`[WI] Removing ${type} entry from timedWorldInfo: entry not ${type}`, entry);
delete chat_metadata.timedWorldInfo[type][key];
continue;
}
if (this.#chat.length >= Number(value.end)) {
console.log(`Removing ${type} entry from timedWorldInfo: ${type} interval passed`, entry);
console.log(`[WI] Removing ${type} entry from timedWorldInfo: ${type} interval passed`, entry);
delete chat_metadata.timedWorldInfo[type][key];
if (typeof onEnded === 'function') {
onEnded(entry);
@ -572,7 +579,7 @@ class WorldInfoTimedEffects {
}
buffer.push(entry);
console.log(`Timed effect "${type}" applied to entry`, entry);
console.log(`[WI] Timed effect "${type}" applied to entry`, entry);
}
}
@ -588,7 +595,7 @@ class WorldInfoTimedEffects {
if (this.#chat.length < entry.delay) {
buffer.push(entry);
console.log('Timed effect "delay" applied to entry', entry);
console.log('[WI] Timed effect "delay" applied to entry', entry);
}
}
@ -635,7 +642,7 @@ class WorldInfoTimedEffects {
const effect = this.#getEntryTimedEffect(type, entry, false);
chat_metadata.timedWorldInfo[type][key] = effect;
console.log(`Adding ${type} entry ${key}: start=${effect.start}, end=${effect.end}, protected=${effect.protected}`);
console.log(`[WI] Adding ${type} entry ${key}: start=${effect.start}, end=${effect.end}, protected=${effect.protected}`);
}
}
@ -667,7 +674,7 @@ class WorldInfoTimedEffects {
if (newState) {
const effect = this.#getEntryTimedEffect(type, entry, false);
chat_metadata.timedWorldInfo[type][key] = effect;
console.log(`Adding ${type} entry ${key}: start=${effect.start}, end=${effect.end}, protected=${effect.protected}`);
console.log(`[WI] Adding ${type} entry ${key}: start=${effect.start}, end=${effect.end}, protected=${effect.protected}`);
}
}
@ -3419,13 +3426,12 @@ async function createNewWorldInfo(worldName, { interactive = false } = {}) {
async function getCharacterLore() {
const character = characters[this_chid];
const name = character?.name;
/** @type {Set<string>} */
let worldsToSearch = new Set();
const baseWorldName = character?.data?.extensions?.world;
if (baseWorldName) {
worldsToSearch.add(baseWorldName);
} else {
console.debug(`Character ${name}'s base world could not be found or is empty! Skipping...`);
}
// TODO: Maybe make the utility function not use the window context?
@ -3435,40 +3441,48 @@ async function getCharacterLore() {
worldsToSearch = new Set([...worldsToSearch, ...extraCharLore.extraBooks]);
}
if (!worldsToSearch.size) {
return [];
}
let entries = [];
for (const worldName of worldsToSearch) {
if (selected_world_info.includes(worldName)) {
console.debug(`Character ${name}'s world ${worldName} is already activated in global world info! Skipping...`);
console.debug(`[WI] Character ${name}'s world ${worldName} is already activated in global world info! Skipping...`);
continue;
}
if (chat_metadata[METADATA_KEY] === worldName) {
console.debug(`Character ${name}'s world ${worldName} is already activated in chat lore! Skipping...`);
console.debug(`[WI] Character ${name}'s world ${worldName} is already activated in chat lore! Skipping...`);
continue;
}
const data = await loadWorldInfoData(worldName);
const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x => ({ ...x, world: worldName })) : [];
const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(({ uid, ...rest }) => ({ uid, world: worldName, ...rest })) : [];
entries = entries.concat(newEntries);
if (!newEntries.length) {
console.debug(`[WI] Character ${name}'s world ${worldName} could not be found or is empty`);
}
}
console.debug(`Character ${name} lore (${Array.from(worldsToSearch)}) has ${entries.length} world info entries`);
console.debug(`[WI] Character ${name}'s lore has ${entries.length} world info entries`, [...worldsToSearch]);
return entries;
}
async function getGlobalLore() {
if (!selected_world_info) {
if (!selected_world_info?.length) {
return [];
}
let entries = [];
for (const worldName of selected_world_info) {
const data = await loadWorldInfoData(worldName);
const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x => ({ ...x, world: worldName })) : [];
const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(({ uid, ...rest }) => ({ uid, world: worldName, ...rest })) : [];
entries = entries.concat(newEntries);
}
console.debug(`Global world info has ${entries.length} entries`);
console.debug(`[WI] Global world info has ${entries.length} entries`, selected_world_info);
return entries;
}
@ -3481,14 +3495,14 @@ async function getChatLore() {
}
if (selected_world_info.includes(chatWorld)) {
console.debug(`Chat world ${chatWorld} is already activated in global world info! Skipping...`);
console.debug(`[WI] Chat world ${chatWorld} is already activated in global world info! Skipping...`);
return [];
}
const data = await loadWorldInfoData(chatWorld);
const entries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x => ({ ...x, world: chatWorld })) : [];
const entries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(({ uid, ...rest }) => ({ uid, world: chatWorld, ...rest })) : [];
console.debug(`Chat lore has ${entries.length} entries`);
console.debug(`[WI] Chat lore has ${entries.length} entries`, [chatWorld]);
return entries;
}
@ -3512,7 +3526,7 @@ export async function getSortedEntries() {
entries = [...globalLore.sort(sortFn), ...characterLore.sort(sortFn)];
break;
default:
console.error('Unknown WI insertion strategy: ', world_info_character_strategy, 'defaulting to evenly');
console.error('[WI] Unknown WI insertion strategy:', world_info_character_strategy, 'defaulting to evenly');
entries = [...globalLore, ...characterLore].sort(sortFn);
break;
}
@ -3520,7 +3534,7 @@ export async function getSortedEntries() {
// Chat lore always goes first
entries = [...chatLore.sort(sortFn), ...entries];
console.debug(`Sorted ${entries.length} world lore entries using strategy ${world_info_character_strategy}`);
console.debug(`[WI] Found ${entries.length} world lore entries. Sorted by strategy`, Object.entries(world_info_insertion_strategy).find((x) => x[1] === world_info_character_strategy));
// Need to deep clone the entries to avoid modifying the cached data
return structuredClone(entries);
@ -3543,6 +3557,8 @@ async function checkWorldInfo(chat, maxContext, isDryRun) {
const context = getContext();
const buffer = new WorldInfoBuffer(chat);
console.debug(`[WI] --- START WI SCAN (on ${chat.length} messages) ---`);
// Combine the chat
// Add the depth or AN if enabled
@ -3566,11 +3582,11 @@ async function checkWorldInfo(chat, maxContext, isDryRun) {
let budget = Math.round(world_info_budget * maxContext / 100) || 1;
if (world_info_budget_cap > 0 && budget > world_info_budget_cap) {
console.debug(`Budget ${budget} exceeds cap ${world_info_budget_cap}, using cap`);
console.debug(`[WI] Budget ${budget} exceeds cap ${world_info_budget_cap}, using cap`);
budget = world_info_budget_cap;
}
console.debug(`Context size: ${maxContext}; WI budget: ${budget} (max% = ${world_info_budget}%, cap = ${world_info_budget_cap})`);
console.debug(`[WI] Context size: ${maxContext}; WI budget: ${budget} (max% = ${world_info_budget}%, cap = ${world_info_budget_cap})`);
const sortedEntries = await getSortedEntries();
const timedEffects = new WorldInfoTimedEffects(chat, sortedEntries);
@ -3580,21 +3596,48 @@ async function checkWorldInfo(chat, maxContext, isDryRun) {
return { worldInfoBefore: '', worldInfoAfter: '', WIDepthEntries: [], EMEntries: [], allActivatedEntries: new Set() };
}
console.debug(`[WI] --- SEARCHING ENTRIES (on ${sortedEntries.length} entries) ---`);
while (scanState) {
// Track how many times the loop has run. May be useful for debugging.
// eslint-disable-next-line no-unused-vars
count++;
console.debug(`[WI] Loop #${count}. Search state`, Object.entries(scan_state).find(x => x[1] === scanState));
// Until decided otherwise, we set the loop to stop scanning after this
let nextScanState = scan_state.NONE;
// Loop and find all entries that can activate here
let activatedNow = new Set();
for (let entry of sortedEntries) {
// Logging preparation
let headerLogged = false;
function log(...args) {
if (!headerLogged) {
console.debug(`[WI] Entry ${entry.uid}`, `from '${entry.world}' processing`, entry);
headerLogged = true;
}
console.debug(`[WI] Entry ${entry.uid}`, ...args);
}
// Already processed, considered and then skipped entries should still be skipped
if (failedProbabilityChecks.has(entry) || allActivatedEntries.has(entry)) {
continue;
}
if (entry.disable == true) {
log('disabled');
continue;
}
// Check if this entry applies to the character or if it's excluded
if (entry.characterFilter && entry.characterFilter?.names?.length > 0) {
const nameIncluded = entry.characterFilter.names.includes(getCharaFilename());
const filtered = entry.characterFilter.isExclude ? nameIncluded : !nameIncluded;
if (filtered) {
console.debug(`WI entry ${entry.uid} filtered out by character`);
log('filtered out by character');
continue;
}
}
@ -3611,7 +3654,7 @@ async function checkWorldInfo(chat, maxContext, isDryRun) {
const filtered = entry.characterFilter.isExclude ? includesTag : !includesTag;
if (filtered) {
console.debug(`WI entry ${entry.uid} filtered out by tag`);
log('filtered out by tag');
continue;
}
}
@ -3623,186 +3666,242 @@ async function checkWorldInfo(chat, maxContext, isDryRun) {
const isDelay = timedEffects.isEffectActive('delay', entry);
if (isDelay) {
console.debug(`WI entry ${entry.uid} suppressed by delay`, entry);
log('suppressed by delay');
continue;
}
if (isCooldown && !isSticky) {
console.debug(`WI entry ${entry.uid} suppressed by cooldown`, entry);
continue;
}
if (failedProbabilityChecks.has(entry)) {
continue;
}
if (allActivatedEntries.has(entry) || entry.disable == true) {
log('suppressed by cooldown');
continue;
}
// Only use checks for recursion flags if the scan step was activated by recursion
if (scanState !== scan_state.RECURSION && entry.delayUntilRecursion) {
console.debug(`WI entry ${entry.uid} suppressed by delay until recursion`, entry);
log('suppressed by delay until recursion');
continue;
}
if (scanState === scan_state.RECURSION && world_info_recursive && entry.excludeRecursion) {
console.debug(`WI entry ${entry.uid} suppressed by exclude recursion`, entry);
log('suppressed by exclude recursion');
continue;
}
if (entry.constant || buffer.isExternallyActivated(entry) || isSticky) {
// Now do checks for immediate activations
if (entry.constant) {
log('activated because of constant');
activatedNow.add(entry);
continue;
}
if (Array.isArray(entry.key) && entry.key.length) { //check for keywords existing
// If selectiveLogic isn't found, assume it's AND, only do this once per entry
const selectiveLogic = entry.selectiveLogic ?? 0;
if (buffer.isExternallyActivated(entry)) {
log('externally activated');
activatedNow.add(entry);
continue;
}
primary: for (let key of entry.key) {
const substituted = substituteParams(key);
const textToScan = buffer.get(entry, scanState);
if (isSticky) {
log('activated because active sticky');
activatedNow.add(entry);
continue;
}
if (substituted && buffer.matchKeys(textToScan, substituted.trim(), entry)) {
console.debug(`WI UID ${entry.uid} found by primary match: ${substituted}.`);
if (!Array.isArray(entry.key) || !entry.key.length) {
log('has no keys defined, skipped');
continue;
}
//selective logic begins
if (
entry.selective && //all entries are selective now
Array.isArray(entry.keysecondary) && //always true
entry.keysecondary.length //ignore empties
) {
console.debug(`WI UID:${entry.uid} found. Checking logic: ${entry.selectiveLogic}`);
let hasAnyMatch = false;
let hasAllMatch = true;
secondary: for (let keysecondary of entry.keysecondary) {
const secondarySubstituted = substituteParams(keysecondary);
const hasSecondaryMatch = secondarySubstituted && buffer.matchKeys(textToScan, secondarySubstituted.trim(), entry);
console.debug(`WI UID:${entry.uid}: Filtering for secondary keyword - "${secondarySubstituted}".`);
// Cache the text to scan before the loop, it won't change its content
const textToScan = buffer.get(entry, scanState);
if (hasSecondaryMatch) {
hasAnyMatch = true;
}
// PRIMARY KEYWORDS
let primaryKeyMatch = entry.key.find(key => {
const substituted = substituteParams(key);
return substituted && buffer.matchKeys(textToScan, substituted.trim(), entry);
});
if (!hasSecondaryMatch) {
hasAllMatch = false;
}
if (!primaryKeyMatch) {
// Don't write logs for simple no-matches
continue;
}
// Simplified AND ANY / NOT ALL if statement. (Proper fix for PR#1356 by Bronya)
// If AND ANY logic and the main checks pass OR if NOT ALL logic and the main checks do not pass
if ((selectiveLogic === world_info_logic.AND_ANY && hasSecondaryMatch) || (selectiveLogic === world_info_logic.NOT_ALL && !hasSecondaryMatch)) {
// Differ both logic statements in the debugger
if (selectiveLogic === world_info_logic.AND_ANY) {
console.debug(`(AND ANY Check) Activating WI Entry ${entry.uid}. Found match for word: ${substituted} ${secondarySubstituted}`);
} else {
console.debug(`(NOT ALL Check) Activating WI Entry ${entry.uid}. Found match for word "${substituted}" without secondary keyword: ${secondarySubstituted}`);
}
activatedNow.add(entry);
break secondary;
}
}
const hasSecondaryKeywords = (
entry.selective && //all entries are selective now
Array.isArray(entry.keysecondary) && //always true
entry.keysecondary.length //ignore empties
);
// Handle NOT ANY logic
if (selectiveLogic === world_info_logic.NOT_ANY && !hasAnyMatch) {
console.debug(`(NOT ANY Check) Activating WI Entry ${entry.uid}, no secondary keywords found.`);
activatedNow.add(entry);
}
if (!hasSecondaryKeywords) {
// Handle cases where secondary is empty
log('activated by primary key match', primaryKeyMatch);
activatedNow.add(entry);
continue;
}
// Handle AND ALL logic
if (selectiveLogic === world_info_logic.AND_ALL && hasAllMatch) {
console.debug(`(AND ALL Check) Activating WI Entry ${entry.uid}, all secondary keywords found.`);
activatedNow.add(entry);
}
} else {
// Handle cases where secondary is empty
console.debug(`WI UID ${entry.uid}: Activated without filter logic.`);
activatedNow.add(entry);
break primary;
}
// SECONDARY KEYWORDS
const selectiveLogic = entry.selectiveLogic ?? 0; // If selectiveLogic isn't found, assume it's AND, only do this once per entry
log('Entry with primary key match', primaryKeyMatch, 'has secondary keywords. Checking with logic logic', Object.entries(world_info_logic).find(x => x[1] === entry.selectiveLogic));
/** @type {() => boolean} */
function matchSecondaryKeys() {
let hasAnyMatch = false;
let hasAllMatch = true;
for (let keysecondary of entry.keysecondary) {
const secondarySubstituted = substituteParams(keysecondary);
const hasSecondaryMatch = secondarySubstituted && buffer.matchKeys(textToScan, secondarySubstituted.trim(), entry);
if (hasSecondaryMatch) hasAnyMatch = true;
if (!hasSecondaryMatch) hasAllMatch = false;
// Simplified AND ANY / NOT ALL if statement. (Proper fix for PR#1356 by Bronya)
// If AND ANY logic and the main checks pass OR if NOT ALL logic and the main checks do not pass
if (selectiveLogic === world_info_logic.AND_ANY && hasSecondaryMatch) {
log('activated. (AND ANY) Found match secondary keyword', secondarySubstituted);
return true;
}
if (selectiveLogic === world_info_logic.NOT_ALL && !hasSecondaryMatch) {
log('activated. (NOT ALL) Found not matching secondary keyword', secondarySubstituted);
return true;
}
}
// Handle NOT ANY logic
if (selectiveLogic === world_info_logic.NOT_ANY && !hasAnyMatch) {
log('activated. (NOT ANY) No secondary keywords found', entry.keysecondary);
return true;
}
// Handle AND ALL logic
if (selectiveLogic === world_info_logic.AND_ALL && hasAllMatch) {
log('activated. (AND ALL) All secondary keywords found', entry.keysecondary);
return true;
}
return false;
}
const matched = matchSecondaryKeys();
if (!matched) {
log('skipped. Secondary keywords not satisfied', entry.keysecondary);
continue;
}
// Success logging was already done inside the function, so just add the entry
activatedNow.add(entry);
continue;
}
scanState = world_info_recursive && activatedNow.size > 0 ? scan_state.RECURSION : scan_state.NONE;
console.debug(`[WI] Search done. Found ${activatedNow.size} possible entries.`);
const newEntries = [...activatedNow]
.sort((a, b) => sortedEntries.indexOf(a) - sortedEntries.indexOf(b));
let newContent = '';
const textToScanTokens = await getTokenCountAsync(allActivatedText);
const probabilityChecksBefore = failedProbabilityChecks.size;
filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanState);
console.debug('-- PROBABILITY CHECKS BEGIN --');
console.debug('[WI] --- PROBABILITY CHECKS ---');
for (const entry of newEntries) {
const rollValue = Math.random() * 100;
if (entry.useProbability && rollValue > entry.probability) {
const isSticky = timedEffects.isEffectActive('sticky', entry);
if (!isSticky) {
console.debug(`WI entry ${entry.uid} ${entry.key} failed probability check, skipping`);
failedProbabilityChecks.add(entry);
continue;
function verifyProbability() {
// If we don't need to roll, it's always true
if (!entry.useProbability || entry.probability === 100) {
console.debug(`WI entry ${entry.uid} does not use probability`);
return true;
}
} else { console.debug(`uid:${entry.uid} passed probability check, inserting to prompt`); }
const isSticky = timedEffects.isEffectActive('sticky', entry);
if (isSticky) {
console.debug(`WI entry ${entry.uid} is sticky, does not need to re-roll probability`);
return true;
}
const rollValue = Math.random() * 100;
if (rollValue <= entry.probability) {
console.debug(`WI entry ${entry.uid} passed probability check of ${entry.probability}%`);
return true;
}
failedProbabilityChecks.add(entry);
return false;
}
const success = verifyProbability();
if (!success) {
console.debug(`WI entry ${entry.uid} failed probability check, removing from activated entries`, entry);
continue;
}
// Substitute macros inline, for both this checking and also future processing
entry.content = substituteParams(entry.content);
newContent += `${entry.content}\n`;
if ((textToScanTokens + (await getTokenCountAsync(newContent))) >= budget) {
console.debug('WI budget reached, stopping');
if (world_info_overflow_alert) {
console.log('Alerting');
console.warn(`[WI] budget of ${budget} reached, stopping after ${allActivatedEntries.size} entries`);
toastr.warning(`World info budget reached after ${allActivatedEntries.size} entries.`, 'World Info');
} else {
console.debug(`[WI] budget of ${budget} reached, stopping after ${allActivatedEntries.size} entries`);
}
scanState = scan_state.NONE;
token_budget_overflowed = true;
break;
}
allActivatedEntries.add(entry);
console.debug('WI entry activated:', entry);
console.debug(`[WI] Entry ${entry.uid} activation successful, adding to prompt`, entry);
}
const probabilityChecksAfter = failedProbabilityChecks.size;
const successfulNewEntries = newEntries.filter(x => !failedProbabilityChecks.has(x));
const successfulNewEntriesForRecursion = successfulNewEntries.filter(x => !x.preventRecursion);
if ((probabilityChecksAfter - probabilityChecksBefore) === activatedNow.size) {
console.debug('WI probability checks failed for all activated entries, stopping');
scanState = scan_state.NONE;
if (!newEntries.length) {
console.debug('[WI] No new entries activated, stopping');
} else if (!successfulNewEntries.length) {
console.debug('[WI] Probability checks failed for all activated entries, stopping');
} else {
console.debug(`[WI] Successfully activated ${successfulNewEntries.length} new entries to prompt. ${allActivatedEntries.size} total entries activated.`, successfulNewEntries);
}
if (newEntries.length === 0) {
console.debug('No new entries activated, stopping');
scanState = scan_state.NONE;
// After processing and rolling entries is done, see if we should continue with normal recursion
if (world_info_recursive && !token_budget_overflowed && successfulNewEntriesForRecursion.length) {
nextScanState = scan_state.RECURSION;
}
// If we are inside min activations scan, and we have recursive buffer, we should do a recursive scan before increasing the buffer again
// There might be recurse-trigger-able entries that match the buffer, so we need to check that
if (world_info_recursive && !token_budget_overflowed && scanState === scan_state.MIN_ACTIVATIONS && buffer.hasRecurse()) {
nextScanState = scan_state.RECURSION;
}
// If scanning is planned to stop, but min activations is set and not satisfied, check if we should continue
const minActivationsNotSatisfied = world_info_min_activations > 0 && (allActivatedEntries.size < world_info_min_activations);
if (!nextScanState && !token_budget_overflowed && minActivationsNotSatisfied) {
console.debug('[WI] --- MIN ACTIVATIONS CHECK ---');
let over_max = (
world_info_min_activations_depth_max > 0 &&
buffer.getDepth() > world_info_min_activations_depth_max
) || (buffer.getDepth() > chat.length);
if (!over_max) {
console.debug(`[WI] Min activations not reached (${allActivatedEntries.size}/${world_info_min_activations}), advancing depth to ${buffer.getDepth() + 1} and checking again`);
nextScanState = scan_state.MIN_ACTIVATIONS; // loop
buffer.advanceScan();
} else {
console.debug(`[WI] Min activations not reached (${allActivatedEntries.size}/${world_info_min_activations}), but reached on of depth. Stopping`);
}
}
// Final check if we should really continue scan, and extend the current WI recurse buffer
scanState = nextScanState;
if (scanState) {
const text = newEntries
.filter(x => !failedProbabilityChecks.has(x))
.filter(x => !x.preventRecursion)
const text = successfulNewEntriesForRecursion
.map(x => x.content).join('\n');
buffer.addRecurse(text);
allActivatedText = (text + '\n' + allActivatedText);
}
// world_info_min_activations
if (!scanState && !token_budget_overflowed) {
if (world_info_min_activations > 0 && (allActivatedEntries.size < world_info_min_activations)) {
let over_max = (
world_info_min_activations_depth_max > 0 &&
buffer.getDepth() > world_info_min_activations_depth_max
) || (buffer.getDepth() > chat.length);
if (!over_max) {
scanState = scan_state.MIN_ACTIVATIONS; // loop
buffer.advanceScanPosition();
}
}
}
}
console.debug('[WI] --- BUILDING PROMPT ---');
// Forward-sorted list of entries for joining
const WIBeforeEntries = [];
const WIAfterEntries = [];
@ -3818,7 +3917,7 @@ async function checkWorldInfo(chat, maxContext, isDryRun) {
const content = getRegexedString(entry.content, regex_placement.WORLD_INFO, { depth: regexDepth, isMarkdown: false, isPrompt: true });
if (!content) {
console.debug('Skipping adding WI entry to prompt due to empty content:', entry);
console.debug(`[WI] Entry ${entry.uid}`, 'skipped adding to prompt due to empty content', entry);
return;
}
@ -3876,6 +3975,9 @@ async function checkWorldInfo(chat, maxContext, isDryRun) {
buffer.resetExternalEffects();
timedEffects.cleanUp();
console.log(`[WI] Adding ${allActivatedEntries.size} entries to prompt`, Array.from(allActivatedEntries));
console.debug('[WI] --- DONE ---');
return { worldInfoBefore, worldInfoAfter, EMEntries, WIDepthEntries, allActivatedEntries };
}
@ -3890,13 +3992,13 @@ function filterGroupsByScoring(groups, buffer, removeEntry, scanState) {
for (const [key, group] of Object.entries(groups)) {
// Group scoring is disabled both globally and for the group entries
if (!world_info_use_group_scoring && !group.some(x => x.useGroupScoring)) {
console.debug(`Skipping group scoring for group '${key}'`);
console.debug(`[WI] Skipping group scoring for group '${key}'`);
continue;
}
const scores = group.map(entry => buffer.getScore(entry, scanState));
const maxScore = Math.max(...scores);
console.debug(`Group '${key}' max score: ${maxScore}`);
console.debug(`[WI] Group '${key}' max score:`, maxScore);
//console.table(group.map((entry, i) => ({ uid: entry.uid, key: JSON.stringify(entry.key), score: scores[i] })));
for (let i = 0; i < group.length; i++) {
@ -3907,7 +4009,7 @@ function filterGroupsByScoring(groups, buffer, removeEntry, scanState) {
}
if (scores[i] < maxScore) {
console.debug(`Removing score loser from inclusion group '${key}' entry '${group[i].uid}'`, group[i]);
console.debug(`[WI] Entry ${group[i].uid}`, `removed as score loser from inclusion group '${key}'`, group[i]);
removeEntry(group[i]);
group.splice(i, 1);
scores.splice(i, 1);
@ -3925,7 +4027,8 @@ function filterGroupsByScoring(groups, buffer, removeEntry, scanState) {
* @param {number} scanState The current scan state
*/
function filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanState) {
console.debug('-- INCLUSION GROUP CHECKS BEGIN --');
console.debug('[WI] --- INCLUSION GROUP CHECKS ---');
const grouped = newEntries.filter(x => x.group).reduce((acc, item) => {
item.group.split(/,\s*/).filter(x => x).forEach(group => {
if (!acc[group]) {
@ -3937,7 +4040,7 @@ function filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanSt
}, {});
if (Object.keys(grouped).length === 0) {
console.debug('No inclusion groups found');
console.debug('[WI] No inclusion groups found');
return;
}
@ -3948,7 +4051,7 @@ function filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanSt
continue;
}
if (logging) console.debug(`Removing loser from inclusion group '${entry.group}' entry '${entry.uid}'`, entry);
if (logging) console.debug(`[WI] Entry ${entry.uid}`, `removed as loser from inclusion group '${entry.group}'`, entry);
removeEntry(entry);
}
}
@ -3956,24 +4059,24 @@ function filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanSt
filterGroupsByScoring(grouped, buffer, removeEntry, scanState);
for (const [key, group] of Object.entries(grouped)) {
console.debug(`Checking inclusion group '${key}' with ${group.length} entries`, group);
console.debug(`[WI] Checking inclusion group '${key}' with ${group.length} entries`, group);
if (Array.from(allActivatedEntries).some(x => x.group === key)) {
console.debug(`Skipping inclusion group check, group already activated '${key}'`);
console.debug(`[WI] Skipping inclusion group check, group '${key}' was already activated`);
// We need to forcefully deactivate all other entries in the group
removeAllBut(group, null, false);
continue;
}
if (!Array.isArray(group) || group.length <= 1) {
console.debug('Skipping inclusion group check, only one entry');
console.debug('[WI] Skipping inclusion group check, only one entry');
continue;
}
// Check for group prio
const prios = group.filter(x => x.groupOverride).sort(sortFn);
if (prios.length) {
console.debug(`Activated inclusion group '${key}' with by prio winner entry '${prios[0].uid}'`, prios[0]);
console.debug(`[WI] Entry ${prios[0].uid}`, `activated as prio winner from inclusion group '${key}'`, prios[0]);
removeAllBut(group, prios[0]);
continue;
}
@ -3988,14 +4091,14 @@ function filterByInclusionGroups(newEntries, allActivatedEntries, buffer, scanSt
currentWeight += (entry.groupWeight ?? DEFAULT_WEIGHT);
if (rollValue <= currentWeight) {
console.debug(`Activated inclusion group '${key}' with roll winner entry '${entry.uid}'`, entry);
console.debug(`[WI] Entry ${entry.uid}`, `activated as roll winner from inclusion group '${key}'`, entry);
winner = entry;
break;
}
}
if (!winner) {
console.debug(`Failed to activate inclusion group '${key}', no winner found`);
console.debug(`[WI] Failed to activate inclusion group '${key}', no winner found`);
continue;
}