Merge pull request #3889 from BismuthGlass/feature/wi_global_matches

World Info chat-independent data matching

public/css/world-info.css

@@ -124,6 +124,10 @@
     cursor: initial;
 }
 
+.world_entry .inline-drawer-header-pointer {
+    cursor: pointer;
+}
+
 .world_entry .killSwitch {
     cursor: pointer;
 }

public/index.html

@@ -6292,6 +6292,54 @@
                                     </label>
                                 </div>
                             </div>
+                            <div class="inline-drawer wide100p flexFlowColumn">
+                                <div class="inline-drawer-toggle inline-drawer-header inline-drawer-header-pointer userSettingsInnerExpandable">
+                                    <strong data-i18n="Additional Matching Sources">Additional Matching Sources</strong>
+                                    <div class="fa-solid fa-circle-chevron-down inline-drawer-icon down"></div>
+                                </div>
+                                <div class="inline-drawer-content flex-container flexFlowRow flexGap10 paddingBottom5px">
+                                    <small class="flex-container flex1 flexFlowColumn">
+                                        <label class="checkbox flex-container alignItemsCenter flexNoGap">
+                                            <input type="checkbox" name="matchCharacterDescription" />
+                                            <span data-i18n="Character Description">
+                                                Character Description
+                                            </span>
+                                        </label>
+                                        <label class="checkbox flex-container alignItemsCenter flexNoGap">
+                                            <input type="checkbox" name="matchCharacterPersonality" />
+                                            <span data-i18n="Character Personality">
+                                                Character Personality
+                                            </span>
+                                        </label>
+                                        <label class="checkbox flex-container alignItemsCenter flexNoGap">
+                                            <input type="checkbox" name="matchScenario" />
+                                            <span data-i18n="Scenario">
+                                                Scenario
+                                            </span>
+                                        </label>
+                                    </small>
+                                    <small class="flex-container flex1 flexFlowColumn">
+                                        <label class="checkbox flex-container alignItemsCenter flexNoGap">
+                                            <input type="checkbox" name="matchPersonaDescription" />
+                                            <span data-i18n="Persona Description">
+                                                Persona Description
+                                            </span>
+                                        </label>
+                                        <label class="checkbox flex-container alignItemsCenter flexNoGap">
+                                            <input type="checkbox" name="matchCharacterDepthPrompt" />
+                                            <span data-i18n="Character's Note">
+                                                Character's Note
+                                            </span>
+                                        </label>
+                                        <label class="checkbox flex-container alignItemsCenter flexNoGap">
+                                            <input type="checkbox" name="matchCreatorNotes" />
+                                            <span data-i18n="Creator's Notes">
+                                                Creator's Notes
+                                            </span>
+                                        </label>
+                                    </small>
+                                </div>
+                            </div>
                         </div>
                     </div>
                 </form>

public/script.js

@@ -2753,6 +2753,7 @@ export function substituteParams(content, _name1, _name2, _original, _group, _re
         environment.charVersion = fields.version || '';
         environment.char_version = fields.version || '';
         environment.charDepthPrompt = fields.charDepthPrompt || '';
+        environment.creatorNotes = fields.creatorNotes || '';
     }
 
     // Must be substituted last so that they're replaced inside {{description}}
@@ -3131,6 +3132,7 @@ export function baseChatReplace(value, name1, name2) {
  * @property {string} jailbreak Jailbreak instructions
  * @property {string} version Character version
  * @property {string} charDepthPrompt Character depth note
+ * @property {string} creatorNotes Character creator notes
  * @returns {CharacterCardFields} Character card fields
  */
 export function getCharacterCardFields({ chid = null } = {}) {
@@ -3146,6 +3148,7 @@ export function getCharacterCardFields({ chid = null } = {}) {
         jailbreak: '',
         version: '',
         charDepthPrompt: '',
+        creatorNotes: '',
     };
     result.persona = baseChatReplace(power_user.persona_description?.trim(), name1, name2);
 
@@ -3164,6 +3167,7 @@ export function getCharacterCardFields({ chid = null } = {}) {
     result.jailbreak = power_user.prefer_character_jailbreak ? baseChatReplace(character.data?.post_history_instructions?.trim(), name1, name2) : '';
     result.version = character.data?.character_version ?? '';
     result.charDepthPrompt = baseChatReplace(character.data?.extensions?.depth_prompt?.prompt?.trim(), name1, name2);
+    result.creatorNotes = baseChatReplace(character.data?.creator_notes?.trim(), name1, name2);
 
     if (selected_group) {
         const groupCards = getGroupCharacterCards(selected_group, Number(currentChid));
@@ -3991,6 +3995,7 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
         system,
         jailbreak,
         charDepthPrompt,
+        creatorNotes,
     } = getCharacterCardFields();
 
     if (main_api !== 'openai') {
@@ -4145,7 +4150,15 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
     // Make quiet prompt available for WIAN
     setExtensionPrompt('QUIET_PROMPT', quiet_prompt || '', extension_prompt_types.IN_PROMPT, 0, true);
     const chatForWI = coreChat.map(x => world_info_include_names ? `${x.name}: ${x.mes}` : x.mes).reverse();
-    const { worldInfoString, worldInfoBefore, worldInfoAfter, worldInfoExamples, worldInfoDepth } = await getWorldInfoPrompt(chatForWI, this_max_context, dryRun);
+    const globalScanData = {
+        personaDescription: persona,
+        characterDescription: description,
+        characterPersonality: personality,
+        characterDepthPrompt: charDepthPrompt,
+        scenario: scenario,
+        creatorNotes: creatorNotes,
+    };
+    const { worldInfoString, worldInfoBefore, worldInfoAfter, worldInfoExamples, worldInfoDepth } = await getWorldInfoPrompt(chatForWI, this_max_context, dryRun, globalScanData);
     setExtensionPrompt('QUIET_PROMPT', '', extension_prompt_types.IN_PROMPT, 0, true);
 
     // Add message example WI

public/scripts/char-data.js

@@ -33,6 +33,12 @@
  * @property {number} role - The specific function or purpose of the extension.
  * @property {boolean} vectorized - Indicates if the extension is optimized for vectorized processing.
  * @property {number} display_index - The order in which the extension should be displayed for user interfaces.
+ * @property {boolean} match_persona_description - Wether to match against the persona description.
+ * @property {boolean} match_character_description - Wether to match against the persona description.
+ * @property {boolean} match_character_personality - Wether to match against the character personality.
+ * @property {boolean} match_character_depth_prompt - Wether to match against the character depth prompt.
+ * @property {boolean} match_scenario - Wether to match against the character scenario.
+ * @property {boolean} match_creator_notes - Wether to match against the character creator notes.
  */
 
 /**

public/scripts/world-info.js

@@ -97,12 +97,29 @@ export const MAX_SCAN_DEPTH = 1000;
 const KNOWN_DECORATORS = ['@@activate', '@@dont_activate'];
 
 // Typedef area
+/**
+ * @typedef {object} WIGlobalScanData The chat-independent data to be scanned. Each of
+ *     these fields can be enabled for scanning per entry.
+ * @property {string} personaDescription User persona description
+ * @property {string} characterDescription Character description
+ * @property {string} characterPersonality Character personality
+ * @property {string} characterDepthPrompt Character depth prompt (sometimes referred to as character notes)
+ * @property {string} scenario Character defined scenario
+ * @property {string} creatorNotes Character creator notes
+ */
+
 /**
  * @typedef {object} WIScanEntry The entry that triggered the scan
  * @property {number} [scanDepth] The depth of the scan
  * @property {boolean} [caseSensitive] If the scan is case sensitive
  * @property {boolean} [matchWholeWords] If the scan should match whole words
  * @property {boolean} [useGroupScoring] If the scan should use group scoring
+ * @property {boolean} [matchPersonaDescription] If the scan should match against the persona description
+ * @property {boolean} [matchCharacterDescription] If the scan should match against the character description
+ * @property {boolean} [matchCharacterPersonality] If the scan should match against the character personality
+ * @property {boolean} [matchCharacterDepthPrompt] If the scan should match against the character depth prompt
+ * @property {boolean} [matchScenario] If the scan should match against the character scenario
+ * @property {boolean} [matchCreatorNotes] If the scan should match against the creator notes
  * @property {number} [uid] The UID of the entry that triggered the scan
  * @property {string} [world] The world info book of origin of the entry
  * @property {string[]} [key] The primary keys to scan for
@@ -138,6 +155,11 @@ class WorldInfoBuffer {
      */
     static externalActivations = new Map();
 
+    /**
+     * @type {WIGlobalScanData} Chat independent data to be scanned, such as persona and character descriptions
+     */
+    #globalScanData = null;
+
     /**
      * @type {string[]} Array of messages sorted by ascending depth
      */
@@ -166,9 +188,11 @@ class WorldInfoBuffer {
     /**
      * Initialize the buffer with the given messages.
      * @param {string[]} messages Array of messages to add to the buffer
+     * @param {WIGlobalScanData} globalScanData Chat independent context to be scanned
      */
-    constructor(messages) {
+    constructor(messages, globalScanData) {
         this.#initDepthBuffer(messages);
+        this.#globalScanData = globalScanData;
     }
 
     /**
@@ -225,6 +249,25 @@ class WorldInfoBuffer {
         const JOINER = '\n' + MATCHER;
         let result = MATCHER + this.#depthBuffer.slice(this.#startDepth, depth).join(JOINER);
 
+        if (entry.matchPersonaDescription && this.#globalScanData.personaDescription) {
+            result += JOINER + this.#globalScanData.personaDescription;
+        }
+        if (entry.matchCharacterDescription && this.#globalScanData.characterDescription) {
+            result += JOINER + this.#globalScanData.characterDescription;
+        }
+        if (entry.matchCharacterPersonality && this.#globalScanData.characterPersonality) {
+            result += JOINER + this.#globalScanData.characterPersonality;
+        }
+        if (entry.matchCharacterDepthPrompt && this.#globalScanData.characterDepthPrompt) {
+            result += JOINER + this.#globalScanData.characterDepthPrompt;
+        }
+        if (entry.matchScenario && this.#globalScanData.scenario) {
+            result += JOINER + this.#globalScanData.scenario;
+        }
+        if (entry.matchCreatorNotes && this.#globalScanData.creatorNotes) {
+            result += JOINER + this.#globalScanData.creatorNotes;
+        }
+
         if (this.#injectBuffer.length > 0) {
             result += JOINER + this.#injectBuffer.join(JOINER);
         }
@@ -756,6 +799,7 @@ export const worldInfoCache = new StructuredCloneMap({ cloneOnGet: true, cloneOn
  * @param {string[]} chat - The chat messages to scan, in reverse order.
  * @param {number} maxContext - The maximum context size of the generation.
  * @param {boolean} isDryRun - If true, the function will not emit any events.
+ * @param {WIGlobalScanData} globalScanData Chat independent context to be scanned
  * @typedef {object} WIPromptResult
  * @property {string} worldInfoString - Complete world info string
  * @property {string} worldInfoBefore - World info that goes before the prompt
@@ -766,10 +810,10 @@ export const worldInfoCache = new StructuredCloneMap({ cloneOnGet: true, cloneOn
  * @property {Array} anAfter - Array of entries after Author's Note
  * @returns {Promise<WIPromptResult>} The world info string and depth.
  */
-export async function getWorldInfoPrompt(chat, maxContext, isDryRun) {
+export async function getWorldInfoPrompt(chat, maxContext, isDryRun, globalScanData) {
     let worldInfoString = '', worldInfoBefore = '', worldInfoAfter = '';
 
-    const activatedWorldInfo = await checkWorldInfo(chat, maxContext, isDryRun);
+    const activatedWorldInfo = await checkWorldInfo(chat, maxContext, isDryRun, globalScanData);
     worldInfoBefore = activatedWorldInfo.worldInfoBefore;
     worldInfoAfter = activatedWorldInfo.worldInfoAfter;
     worldInfoString = worldInfoBefore + worldInfoAfter;
@@ -2191,6 +2235,12 @@ export const originalWIDataKeyMap = {
     'matchWholeWords': 'extensions.match_whole_words',
     'useGroupScoring': 'extensions.use_group_scoring',
     'caseSensitive': 'extensions.case_sensitive',
+    'matchPersonaDescription': 'extensions.match_persona_description',
+    'matchCharacterDescription': 'extensions.match_character_description',
+    'matchCharacterPersonality': 'extensions.match_character_personality',
+    'matchCharacterDepthPrompt': 'extensions.match_character_depth_prompt',
+    'matchScenario': 'extensions.match_scenario',
+    'matchCreatorNotes': 'extensions.match_creator_notes',
     'scanDepth': 'extensions.scan_depth',
     'automationId': 'extensions.automation_id',
     'vectorized': 'extensions.vectorized',
@@ -3308,6 +3358,28 @@ export async function getWorldEntry(name, data, entry) {
     });
     useGroupScoringSelect.val((entry.useGroupScoring === null || entry.useGroupScoring === undefined) ? 'null' : entry.useGroupScoring ? 'true' : 'false').trigger('input');
 
+    function handleMatchCheckbox(fieldName) {
+        const key = originalWIDataKeyMap[fieldName];
+        const checkBoxElem = template.find(`input[type="checkbox"][name="${fieldName}"]`);
+        checkBoxElem.data('uid', entry.uid);
+        checkBoxElem.on('input', async function () {
+            const uid = $(this).data('uid');
+            const value = $(this).prop('checked');
+
+            data.entries[uid][fieldName] = value;
+            setWIOriginalDataValue(data, uid, key, data.entries[uid][fieldName]);
+            await saveWorldInfo(name, data);
+        });
+        checkBoxElem.prop('checked', !!entry[fieldName]).trigger('input');
+    }
+
+    handleMatchCheckbox('matchPersonaDescription');
+    handleMatchCheckbox('matchCharacterDescription');
+    handleMatchCheckbox('matchCharacterPersonality');
+    handleMatchCheckbox('matchCharacterDepthPrompt');
+    handleMatchCheckbox('matchScenario');
+    handleMatchCheckbox('matchCreatorNotes');
+
     // automation id
     const automationIdInput = template.find('input[name="automationId"]');
     automationIdInput.data('uid', entry.uid);
@@ -3514,6 +3586,12 @@ export const newWorldInfoEntryDefinition = {
     disable: { default: false, type: 'boolean' },
     excludeRecursion: { default: false, type: 'boolean' },
     preventRecursion: { default: false, type: 'boolean' },
+    matchPersonaDescription: { default: false, type: 'boolean' },
+    matchCharacterDescription: { default: false, type: 'boolean' },
+    matchCharacterPersonality: { default: false, type: 'boolean' },
+    matchCharacterDepthPrompt: { default: false, type: 'boolean' },
+    matchScenario: { default: false, type: 'boolean' },
+    matchCreatorNotes: { default: false, type: 'boolean' },
     delayUntilRecursion: { default: 0, type: 'number' },
     probability: { default: 100, type: 'number' },
     useProbability: { default: true, type: 'boolean' },
@@ -3978,6 +4056,7 @@ function parseDecorators(content) {
  * @param {string[]} chat The chat messages to scan, in reverse order.
  * @param {number} maxContext The maximum context size of the generation.
  * @param {boolean} isDryRun Whether to perform a dry run.
+ * @param {WIGlobalScanData} globalScanData Chat independent context to be scanned
  * @typedef {object} WIActivated
  * @property {string} worldInfoBefore The world info before the chat.
  * @property {string} worldInfoAfter The world info after the chat.
@@ -3988,9 +4067,9 @@ function parseDecorators(content) {
  * @property {Set<any>} allActivatedEntries All entries.
  * @returns {Promise<WIActivated>} The world info activated.
  */
-export async function checkWorldInfo(chat, maxContext, isDryRun) {
+export async function checkWorldInfo(chat, maxContext, isDryRun, globalScanData) {
     const context = getContext();
-    const buffer = new WorldInfoBuffer(chat);
+    const buffer = new WorldInfoBuffer(chat, globalScanData);
 
     console.debug(`[WI] --- START WI SCAN (on ${chat.length} messages)${isDryRun ? ' (DRY RUN)' : ''} ---`);
 
@@ -4848,6 +4927,12 @@ export function convertCharacterBook(characterBook) {
             sticky: entry.extensions?.sticky ?? null,
             cooldown: entry.extensions?.cooldown ?? null,
             delay: entry.extensions?.delay ?? null,
+            matchPersonaDescription: entry.extensions?.match_persona_description ?? false,
+            matchCharacterDescription: entry.extensions?.match_character_description ?? false,
+            matchCharacterPersonality: entry.extensions?.match_character_personality ?? false,
+            matchCharacterDepthPrompt: entry.extensions?.match_character_depth_prompt ?? false,
+            matchScenario: entry.extensions?.match_scenario ?? false,
+            matchCreatorNotes: entry.extensions?.match_creator_notes ?? false,
             extensions: entry.extensions ?? {},
         };
     });

src/endpoints/characters.js

@@ -702,6 +702,12 @@ function convertWorldInfoToCharacterBook(name, entries) {
                 sticky: entry.sticky ?? null,
                 cooldown: entry.cooldown ?? null,
                 delay: entry.delay ?? null,
+                match_persona_description: entry.matchPersonaDescription ?? false,
+                match_character_description: entry.matchCharacterDescription ?? false,
+                match_character_personality: entry.matchCharacterPersonality ?? false,
+                match_character_depth_prompt: entry.matchCharacterDepthPrompt ?? false,
+                match_scenario: entry.matchScenario ?? false,
+                match_creator_notes: entry.matchCreatorNotes ?? false,
             },
         };