From eb29f03ab01072eae8cd860608869f3756bcde9f Mon Sep 17 00:00:00 2001 From: Joe Date: Sat, 23 Nov 2024 20:11:46 -0800 Subject: [PATCH 01/11] Sped up character search by 93% --- public/scripts/power-user.js | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js index 43c26bf21..088a56a10 100644 --- a/public/scripts/power-user.js +++ b/public/scripts/power-user.js @@ -330,6 +330,12 @@ const debug_functions = []; const setHotswapsDebounced = debounce(favsToHotswap); +const fuzzySearchCharactersCache = new Map(); +const fuzzySearchWorldInfoCache = new Map(); +const fuzzySearchPersonasCache = new Map(); +const fuzzySearchTagsCache = new Map(); +const fuzzySearchGroupsCache = new Map(); + function playMessageSound() { if (!power_user.play_message_sound) { return; @@ -1830,6 +1836,11 @@ async function loadContextSettings() { * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ export function fuzzySearchCharacters(searchValue) { + + if (fuzzySearchCharactersCache.has(searchValue)) { + return fuzzySearchCharactersCache.get(searchValue); + } + // @ts-ignore const fuse = new Fuse(characters, { keys: [ @@ -1853,6 +1864,7 @@ export function fuzzySearchCharacters(searchValue) { const results = fuse.search(searchValue); console.debug('Characters fuzzy search results for ' + searchValue, results); + fuzzySearchCharactersCache.set(searchValue, results); return results; } @@ -1863,6 +1875,10 @@ export function fuzzySearchCharacters(searchValue) { * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ export function fuzzySearchWorldInfo(data, searchValue) { + if (fuzzySearchWorldInfoCache.has(searchValue)) { + return fuzzySearchWorldInfoCache.get(searchValue); + } + // @ts-ignore const fuse = new Fuse(data, { keys: [ @@ -1882,6 +1898,7 @@ export function fuzzySearchWorldInfo(data, searchValue) { const results = fuse.search(searchValue); console.debug('World Info fuzzy search results for ' + searchValue, results); + fuzzySearchWorldInfoCache.set(searchValue, results); return results; } @@ -1892,6 +1909,10 @@ export function fuzzySearchWorldInfo(data, searchValue) { * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ export function fuzzySearchPersonas(data, searchValue) { + if (fuzzySearchPersonasCache.has(searchValue)) { + return fuzzySearchPersonasCache.get(searchValue); + } + data = data.map(x => ({ key: x, name: power_user.personas[x] ?? '', description: power_user.persona_descriptions[x]?.description ?? '' })); // @ts-ignore const fuse = new Fuse(data, { @@ -1907,6 +1928,7 @@ export function fuzzySearchPersonas(data, searchValue) { const results = fuse.search(searchValue); console.debug('Personas fuzzy search results for ' + searchValue, results); + fuzzySearchPersonasCache.set(searchValue, results); return results; } @@ -1916,6 +1938,10 @@ export function fuzzySearchPersonas(data, searchValue) { * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ export function fuzzySearchTags(searchValue) { + if (fuzzySearchTagsCache.has(searchValue)) { + return fuzzySearchTagsCache.get(searchValue); + } + // @ts-ignore const fuse = new Fuse(tags, { keys: [ @@ -1929,6 +1955,7 @@ export function fuzzySearchTags(searchValue) { const results = fuse.search(searchValue); console.debug('Tags fuzzy search results for ' + searchValue, results); + fuzzySearchTagsCache.set(searchValue, results); return results; } @@ -1938,6 +1965,10 @@ export function fuzzySearchTags(searchValue) { * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ export function fuzzySearchGroups(searchValue) { + if (fuzzySearchGroupsCache.has(searchValue)) { + return fuzzySearchGroupsCache.get(searchValue); + } + // @ts-ignore const fuse = new Fuse(groups, { keys: [ @@ -1954,6 +1985,7 @@ export function fuzzySearchGroups(searchValue) { const results = fuse.search(searchValue); console.debug('Groups fuzzy search results for ' + searchValue, results); + fuzzySearchGroupsCache.set(searchValue, results); return results; } From 2661881bc4c95f752786b35daa06af3cffe05e15 Mon Sep 17 00:00:00 2001 From: Joe Date: Sat, 23 Nov 2024 22:13:20 -0800 Subject: [PATCH 02/11] Handle stale cache --- public/scripts/power-user.js | 228 +++++++++++++++++++++++++---------- 1 file changed, 165 insertions(+), 63 deletions(-) diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js index 088a56a10..fef259016 100644 --- a/public/scripts/power-user.js +++ b/public/scripts/power-user.js @@ -330,12 +330,6 @@ const debug_functions = []; const setHotswapsDebounced = debounce(favsToHotswap); -const fuzzySearchCharactersCache = new Map(); -const fuzzySearchWorldInfoCache = new Map(); -const fuzzySearchPersonasCache = new Map(); -const fuzzySearchTagsCache = new Map(); -const fuzzySearchGroupsCache = new Map(); - function playMessageSound() { if (!power_user.play_message_sound) { return; @@ -1830,32 +1824,70 @@ async function loadContextSettings() { }); } +// Create separate caches for each type of fuzzy search +const fuzzySearchCaches = { + characters: { keyHash: null, resultMap: new Map() }, + worldInfo: { keyHash: null, resultMap: new Map() }, + personas: { keyHash: null, resultMap: new Map() }, + tags: { keyHash: null, resultMap: new Map() }, + groups: { keyHash: null, resultMap: new Map() }, +}; + /** - * Fuzzy search characters by a search term + * Generates a hash of the Fuse configuration keys + * @param {Array<{name: string, weight: number, getFn?: Function}>} keys + * @returns {number} Hash of the keys configuration + */ +function hashFuseKeys(keys) { + // Convert keys to a stable string representation + const keyString = keys.map(k => { + const getFnString = k.getFn ? k.getFn.toString() : ''; + return `${k.name}:${k.weight}:${getFnString}`; + }).join('|'); + + return getStringHash(keyString); +} + +/** + * Fuzzy search characters by a search term with caching * @param {string} searchValue - The search term * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ export function fuzzySearchCharacters(searchValue) { + const fuseKeys = [ + { name: 'data.name', weight: 20 }, + { name: '#tags', weight: 10, getFn: (character) => getTagsList(character.avatar).map(x => x.name).join('||') }, + { name: 'data.description', weight: 3 }, + { name: 'data.mes_example', weight: 3 }, + { name: 'data.scenario', weight: 2 }, + { name: 'data.personality', weight: 2 }, + { name: 'data.first_mes', weight: 2 }, + { name: 'data.creator_notes', weight: 2 }, + { name: 'data.creator', weight: 1 }, + { name: 'data.tags', weight: 1 }, + { name: 'data.alternate_greetings', weight: 1 }, + ]; - if (fuzzySearchCharactersCache.has(searchValue)) { - return fuzzySearchCharactersCache.get(searchValue); + // Generate hash of current keys configuration + const currentKeyHash = hashFuseKeys(fuseKeys); + const cache = fuzzySearchCaches.characters; + + // Check if we have a cached result for this search value + if (cache.keyHash === currentKeyHash && cache.resultMap.has(searchValue)) { + console.debug('Using cached fuzzy search results for ' + searchValue); + return cache.resultMap.get(searchValue); } - // @ts-ignore + // If key configuration changed, clear the cache + if (cache.keyHash !== currentKeyHash) { + console.debug('Fuse keys changed, clearing cache'); + cache.keyHash = currentKeyHash; + cache.resultMap.clear(); + } + + // Create new Fuse instance and perform search const fuse = new Fuse(characters, { - keys: [ - { name: 'data.name', weight: 20 }, - { name: '#tags', weight: 10, getFn: (character) => getTagsList(character.avatar).map(x => x.name).join('||') }, - { name: 'data.description', weight: 3 }, - { name: 'data.mes_example', weight: 3 }, - { name: 'data.scenario', weight: 2 }, - { name: 'data.personality', weight: 2 }, - { name: 'data.first_mes', weight: 2 }, - { name: 'data.creator_notes', weight: 2 }, - { name: 'data.creator', weight: 1 }, - { name: 'data.tags', weight: 1 }, - { name: 'data.alternate_greetings', weight: 1 }, - ], + keys: fuseKeys, includeScore: true, ignoreLocation: true, useExtendedSearch: true, @@ -1863,8 +1895,11 @@ export function fuzzySearchCharacters(searchValue) { }); const results = fuse.search(searchValue); + + // Cache the results + cache.resultMap.set(searchValue, results); + console.debug('Characters fuzzy search results for ' + searchValue, results); - fuzzySearchCharactersCache.set(searchValue, results); return results; } @@ -1875,21 +1910,34 @@ export function fuzzySearchCharacters(searchValue) { * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ export function fuzzySearchWorldInfo(data, searchValue) { - if (fuzzySearchWorldInfoCache.has(searchValue)) { - return fuzzySearchWorldInfoCache.get(searchValue); + const fuseKeys = [ + { name: 'key', weight: 20 }, + { name: 'group', weight: 15 }, + { name: 'comment', weight: 10 }, + { name: 'keysecondary', weight: 10 }, + { name: 'content', weight: 3 }, + { name: 'uid', weight: 1 }, + { name: 'automationId', weight: 1 }, + ]; + + const currentKeyHash = hashFuseKeys(fuseKeys); + const cache = fuzzySearchCaches.worldInfo; + + // Check cache for existing results + if (cache.keyHash === currentKeyHash && cache.resultMap.has(searchValue)) { + console.debug('Using cached WI fuzzy search results for ' + searchValue); + return cache.resultMap.get(searchValue); + } + + // Clear cache if keys changed + if (cache.keyHash !== currentKeyHash) { + console.debug('WI Fuse keys changed, clearing cache'); + cache.keyHash = currentKeyHash; + cache.resultMap.clear(); } - // @ts-ignore const fuse = new Fuse(data, { - keys: [ - { name: 'key', weight: 20 }, - { name: 'group', weight: 15 }, - { name: 'comment', weight: 10 }, - { name: 'keysecondary', weight: 10 }, - { name: 'content', weight: 3 }, - { name: 'uid', weight: 1 }, - { name: 'automationId', weight: 1 }, - ], + keys: fuseKeys, includeScore: true, ignoreLocation: true, useExtendedSearch: true, @@ -1897,8 +1945,9 @@ export function fuzzySearchWorldInfo(data, searchValue) { }); const results = fuse.search(searchValue); + cache.resultMap.set(searchValue, results); + console.debug('World Info fuzzy search results for ' + searchValue, results); - fuzzySearchWorldInfoCache.set(searchValue, results); return results; } @@ -1909,17 +1958,31 @@ export function fuzzySearchWorldInfo(data, searchValue) { * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ export function fuzzySearchPersonas(data, searchValue) { - if (fuzzySearchPersonasCache.has(searchValue)) { - return fuzzySearchPersonasCache.get(searchValue); + data = data.map(x => ({ key: x, name: power_user.personas[x] ?? '', description: power_user.persona_descriptions[x]?.description ?? '' })); + + const fuseKeys = [ + { name: 'name', weight: 20 }, + { name: 'description', weight: 3 }, + ]; + + const currentKeyHash = hashFuseKeys(fuseKeys); + const cache = fuzzySearchCaches.personas; + + // Check cache for existing results + if (cache.keyHash === currentKeyHash && cache.resultMap.has(searchValue)) { + console.debug('Using cached personas fuzzy search results for ' + searchValue); + return cache.resultMap.get(searchValue); + } + + // Clear cache if keys changed + if (cache.keyHash !== currentKeyHash) { + console.debug('Personas Fuse keys changed, clearing cache'); + cache.keyHash = currentKeyHash; + cache.resultMap.clear(); } - data = data.map(x => ({ key: x, name: power_user.personas[x] ?? '', description: power_user.persona_descriptions[x]?.description ?? '' })); - // @ts-ignore const fuse = new Fuse(data, { - keys: [ - { name: 'name', weight: 20 }, - { name: 'description', weight: 3 }, - ], + keys: fuseKeys, includeScore: true, ignoreLocation: true, useExtendedSearch: true, @@ -1927,8 +1990,9 @@ export function fuzzySearchPersonas(data, searchValue) { }); const results = fuse.search(searchValue); + cache.resultMap.set(searchValue, results); + console.debug('Personas fuzzy search results for ' + searchValue, results); - fuzzySearchPersonasCache.set(searchValue, results); return results; } @@ -1938,15 +2002,28 @@ export function fuzzySearchPersonas(data, searchValue) { * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ export function fuzzySearchTags(searchValue) { - if (fuzzySearchTagsCache.has(searchValue)) { - return fuzzySearchTagsCache.get(searchValue); + const fuseKeys = [ + { name: 'name', weight: 1 }, + ]; + + const currentKeyHash = hashFuseKeys(fuseKeys); + const cache = fuzzySearchCaches.tags; + + // Check cache for existing results + if (cache.keyHash === currentKeyHash && cache.resultMap.has(searchValue)) { + console.debug('Using cached tags fuzzy search results for ' + searchValue); + return cache.resultMap.get(searchValue); + } + + // Clear cache if keys changed + if (cache.keyHash !== currentKeyHash) { + console.debug('Tags Fuse keys changed, clearing cache'); + cache.keyHash = currentKeyHash; + cache.resultMap.clear(); } - // @ts-ignore const fuse = new Fuse(tags, { - keys: [ - { name: 'name', weight: 1 }, - ], + keys: fuseKeys, includeScore: true, ignoreLocation: true, useExtendedSearch: true, @@ -1954,8 +2031,9 @@ export function fuzzySearchTags(searchValue) { }); const results = fuse.search(searchValue); + cache.resultMap.set(searchValue, results); + console.debug('Tags fuzzy search results for ' + searchValue, results); - fuzzySearchTagsCache.set(searchValue, results); return results; } @@ -1965,18 +2043,31 @@ export function fuzzySearchTags(searchValue) { * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ export function fuzzySearchGroups(searchValue) { - if (fuzzySearchGroupsCache.has(searchValue)) { - return fuzzySearchGroupsCache.get(searchValue); + const fuseKeys = [ + { name: 'name', weight: 20 }, + { name: 'members', weight: 15 }, + { name: '#tags', weight: 10, getFn: (group) => getTagsList(group.id).map(x => x.name).join('||') }, + { name: 'id', weight: 1 }, + ]; + + const currentKeyHash = hashFuseKeys(fuseKeys); + const cache = fuzzySearchCaches.groups; + + // Check cache for existing results + if (cache.keyHash === currentKeyHash && cache.resultMap.has(searchValue)) { + console.debug('Using cached groups fuzzy search results for ' + searchValue); + return cache.resultMap.get(searchValue); + } + + // Clear cache if keys changed + if (cache.keyHash !== currentKeyHash) { + console.debug('Groups Fuse keys changed, clearing cache'); + cache.keyHash = currentKeyHash; + cache.resultMap.clear(); } - // @ts-ignore const fuse = new Fuse(groups, { - keys: [ - { name: 'name', weight: 20 }, - { name: 'members', weight: 15 }, - { name: '#tags', weight: 10, getFn: (group) => getTagsList(group.id).map(x => x.name).join('||') }, - { name: 'id', weight: 1 }, - ], + keys: fuseKeys, includeScore: true, ignoreLocation: true, useExtendedSearch: true, @@ -1984,11 +2075,22 @@ export function fuzzySearchGroups(searchValue) { }); const results = fuse.search(searchValue); + cache.resultMap.set(searchValue, results); + console.debug('Groups fuzzy search results for ' + searchValue, results); - fuzzySearchGroupsCache.set(searchValue, results); return results; } +/** + * Clears all fuzzy search caches + */ +export function clearFuzzySearchCaches() { + for (const cache of Object.values(fuzzySearchCaches)) { + cache.keyHash = null; + cache.resultMap.clear(); + } +} + /** * Renders a story string template with the given parameters. * @param {object} params Template parameters. From e2c083ba31d0c9e3ab1e168995f03c0e9d37daac Mon Sep 17 00:00:00 2001 From: Joe Date: Sat, 23 Nov 2024 22:34:56 -0800 Subject: [PATCH 03/11] Remove duplicated code --- public/scripts/power-user.js | 257 +++++++++++------------------------ 1 file changed, 82 insertions(+), 175 deletions(-) diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js index fef259016..9d6910a3f 100644 --- a/public/scripts/power-user.js +++ b/public/scripts/power-user.js @@ -328,6 +328,22 @@ const contextControls = [ let browser_has_focus = true; const debug_functions = []; +const fuzzySearchCaches = { + characters: { keyHash: null, resultMap: new Map() }, + worldInfo: { keyHash: null, resultMap: new Map() }, + personas: { keyHash: null, resultMap: new Map() }, + tags: { keyHash: null, resultMap: new Map() }, + groups: { keyHash: null, resultMap: new Map() }, +}; + +const fuzzySearchCategories = { + characters: 'characters', + worldInfo: 'worldInfo', + personas: 'personas', + tags: 'tags', + groups: 'groups', +}; + const setHotswapsDebounced = debounce(favsToHotswap); function playMessageSound() { @@ -1824,15 +1840,6 @@ async function loadContextSettings() { }); } -// Create separate caches for each type of fuzzy search -const fuzzySearchCaches = { - characters: { keyHash: null, resultMap: new Map() }, - worldInfo: { keyHash: null, resultMap: new Map() }, - personas: { keyHash: null, resultMap: new Map() }, - tags: { keyHash: null, resultMap: new Map() }, - groups: { keyHash: null, resultMap: new Map() }, -}; - /** * Generates a hash of the Fuse configuration keys * @param {Array<{name: string, weight: number, getFn?: Function}>} keys @@ -1849,12 +1856,62 @@ function hashFuseKeys(keys) { } /** - * Fuzzy search characters by a search term with caching + * Common function to perform fuzzy search with caching + * @param {string} type - Type of search from fuzzySearchCategories + * @param {any[]} data - Data array to search in + * @param {Array<{name: string, weight: number, getFn?: Function}>} keys - Fuse.js keys configuration + * @param {string} searchValue - The search term + * @returns {import('fuse.js').FuseResult[]} Results as items with their score + */ +function performFuzzySearch(type, data, keys, searchValue) { + const currentKeyHash = hashFuseKeys(keys); + const cache = fuzzySearchCaches[type]; + + // Check cache for existing results + if (cache.keyHash === currentKeyHash && cache.resultMap.has(searchValue)) { + console.log(`Using cached ${type} fuzzy search results for ${searchValue}`); + return cache.resultMap.get(searchValue); + } + + // Clear cache if keys changed + if (cache.keyHash !== currentKeyHash) { + console.log(`${type} Fuse keys changed, clearing cache`); + cache.keyHash = currentKeyHash; + cache.resultMap.clear(); + } + + const fuse = new Fuse(data, { + keys: keys, + includeScore: true, + ignoreLocation: true, + useExtendedSearch: true, + threshold: 0.2, + }); + + const results = fuse.search(searchValue); + cache.resultMap.set(searchValue, results); + + console.log(`${type} fuzzy search results for ${searchValue}`, results); + return results; +} + +/** + * Clears all fuzzy search caches + */ +export function clearFuzzySearchCaches() { + for (const cache of Object.values(fuzzySearchCaches)) { + cache.keyHash = null; + cache.resultMap.clear(); + } +} + +/** + * Fuzzy search characters by a search term * @param {string} searchValue - The search term * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ export function fuzzySearchCharacters(searchValue) { - const fuseKeys = [ + const keys = [ { name: 'data.name', weight: 20 }, { name: '#tags', weight: 10, getFn: (character) => getTagsList(character.avatar).map(x => x.name).join('||') }, { name: 'data.description', weight: 3 }, @@ -1868,39 +1925,7 @@ export function fuzzySearchCharacters(searchValue) { { name: 'data.alternate_greetings', weight: 1 }, ]; - // Generate hash of current keys configuration - const currentKeyHash = hashFuseKeys(fuseKeys); - const cache = fuzzySearchCaches.characters; - - // Check if we have a cached result for this search value - if (cache.keyHash === currentKeyHash && cache.resultMap.has(searchValue)) { - console.debug('Using cached fuzzy search results for ' + searchValue); - return cache.resultMap.get(searchValue); - } - - // If key configuration changed, clear the cache - if (cache.keyHash !== currentKeyHash) { - console.debug('Fuse keys changed, clearing cache'); - cache.keyHash = currentKeyHash; - cache.resultMap.clear(); - } - - // Create new Fuse instance and perform search - const fuse = new Fuse(characters, { - keys: fuseKeys, - includeScore: true, - ignoreLocation: true, - useExtendedSearch: true, - threshold: 0.2, - }); - - const results = fuse.search(searchValue); - - // Cache the results - cache.resultMap.set(searchValue, results); - - console.debug('Characters fuzzy search results for ' + searchValue, results); - return results; + return performFuzzySearch(fuzzySearchCategories.characters, characters, keys, searchValue); } /** @@ -1910,7 +1935,7 @@ export function fuzzySearchCharacters(searchValue) { * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ export function fuzzySearchWorldInfo(data, searchValue) { - const fuseKeys = [ + const keys = [ { name: 'key', weight: 20 }, { name: 'group', weight: 15 }, { name: 'comment', weight: 10 }, @@ -1920,35 +1945,7 @@ export function fuzzySearchWorldInfo(data, searchValue) { { name: 'automationId', weight: 1 }, ]; - const currentKeyHash = hashFuseKeys(fuseKeys); - const cache = fuzzySearchCaches.worldInfo; - - // Check cache for existing results - if (cache.keyHash === currentKeyHash && cache.resultMap.has(searchValue)) { - console.debug('Using cached WI fuzzy search results for ' + searchValue); - return cache.resultMap.get(searchValue); - } - - // Clear cache if keys changed - if (cache.keyHash !== currentKeyHash) { - console.debug('WI Fuse keys changed, clearing cache'); - cache.keyHash = currentKeyHash; - cache.resultMap.clear(); - } - - const fuse = new Fuse(data, { - keys: fuseKeys, - includeScore: true, - ignoreLocation: true, - useExtendedSearch: true, - threshold: 0.2, - }); - - const results = fuse.search(searchValue); - cache.resultMap.set(searchValue, results); - - console.debug('World Info fuzzy search results for ' + searchValue, results); - return results; + return performFuzzySearch(fuzzySearchCategories.worldInfo, data, keys, searchValue); } /** @@ -1958,42 +1955,18 @@ export function fuzzySearchWorldInfo(data, searchValue) { * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ export function fuzzySearchPersonas(data, searchValue) { - data = data.map(x => ({ key: x, name: power_user.personas[x] ?? '', description: power_user.persona_descriptions[x]?.description ?? '' })); + const mappedData = data.map(x => ({ + key: x, + name: power_user.personas[x] ?? '', + description: power_user.persona_descriptions[x]?.description ?? '' + })); - const fuseKeys = [ + const keys = [ { name: 'name', weight: 20 }, { name: 'description', weight: 3 }, ]; - const currentKeyHash = hashFuseKeys(fuseKeys); - const cache = fuzzySearchCaches.personas; - - // Check cache for existing results - if (cache.keyHash === currentKeyHash && cache.resultMap.has(searchValue)) { - console.debug('Using cached personas fuzzy search results for ' + searchValue); - return cache.resultMap.get(searchValue); - } - - // Clear cache if keys changed - if (cache.keyHash !== currentKeyHash) { - console.debug('Personas Fuse keys changed, clearing cache'); - cache.keyHash = currentKeyHash; - cache.resultMap.clear(); - } - - const fuse = new Fuse(data, { - keys: fuseKeys, - includeScore: true, - ignoreLocation: true, - useExtendedSearch: true, - threshold: 0.2, - }); - - const results = fuse.search(searchValue); - cache.resultMap.set(searchValue, results); - - console.debug('Personas fuzzy search results for ' + searchValue, results); - return results; + return performFuzzySearch(fuzzySearchCategories.personas, mappedData, keys, searchValue); } /** @@ -2002,39 +1975,11 @@ export function fuzzySearchPersonas(data, searchValue) { * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ export function fuzzySearchTags(searchValue) { - const fuseKeys = [ + const keys = [ { name: 'name', weight: 1 }, ]; - const currentKeyHash = hashFuseKeys(fuseKeys); - const cache = fuzzySearchCaches.tags; - - // Check cache for existing results - if (cache.keyHash === currentKeyHash && cache.resultMap.has(searchValue)) { - console.debug('Using cached tags fuzzy search results for ' + searchValue); - return cache.resultMap.get(searchValue); - } - - // Clear cache if keys changed - if (cache.keyHash !== currentKeyHash) { - console.debug('Tags Fuse keys changed, clearing cache'); - cache.keyHash = currentKeyHash; - cache.resultMap.clear(); - } - - const fuse = new Fuse(tags, { - keys: fuseKeys, - includeScore: true, - ignoreLocation: true, - useExtendedSearch: true, - threshold: 0.2, - }); - - const results = fuse.search(searchValue); - cache.resultMap.set(searchValue, results); - - console.debug('Tags fuzzy search results for ' + searchValue, results); - return results; + return performFuzzySearch(fuzzySearchCategories.tags, tags, keys, searchValue); } /** @@ -2043,52 +1988,14 @@ export function fuzzySearchTags(searchValue) { * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ export function fuzzySearchGroups(searchValue) { - const fuseKeys = [ + const keys = [ { name: 'name', weight: 20 }, { name: 'members', weight: 15 }, { name: '#tags', weight: 10, getFn: (group) => getTagsList(group.id).map(x => x.name).join('||') }, { name: 'id', weight: 1 }, ]; - const currentKeyHash = hashFuseKeys(fuseKeys); - const cache = fuzzySearchCaches.groups; - - // Check cache for existing results - if (cache.keyHash === currentKeyHash && cache.resultMap.has(searchValue)) { - console.debug('Using cached groups fuzzy search results for ' + searchValue); - return cache.resultMap.get(searchValue); - } - - // Clear cache if keys changed - if (cache.keyHash !== currentKeyHash) { - console.debug('Groups Fuse keys changed, clearing cache'); - cache.keyHash = currentKeyHash; - cache.resultMap.clear(); - } - - const fuse = new Fuse(groups, { - keys: fuseKeys, - includeScore: true, - ignoreLocation: true, - useExtendedSearch: true, - threshold: 0.2, - }); - - const results = fuse.search(searchValue); - cache.resultMap.set(searchValue, results); - - console.debug('Groups fuzzy search results for ' + searchValue, results); - return results; -} - -/** - * Clears all fuzzy search caches - */ -export function clearFuzzySearchCaches() { - for (const cache of Object.values(fuzzySearchCaches)) { - cache.keyHash = null; - cache.resultMap.clear(); - } + return performFuzzySearch(fuzzySearchCategories.groups, groups, keys, searchValue); } /** From d1cac298c61601449a42b0021b5b860b83c9d8e2 Mon Sep 17 00:00:00 2001 From: Joe Date: Sat, 23 Nov 2024 22:51:48 -0800 Subject: [PATCH 04/11] Coment out log statements --- public/scripts/power-user.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js index 9d6910a3f..8b9e837c2 100644 --- a/public/scripts/power-user.js +++ b/public/scripts/power-user.js @@ -1869,13 +1869,13 @@ function performFuzzySearch(type, data, keys, searchValue) { // Check cache for existing results if (cache.keyHash === currentKeyHash && cache.resultMap.has(searchValue)) { - console.log(`Using cached ${type} fuzzy search results for ${searchValue}`); + // console.debug(`Using cached ${type} fuzzy search results for ${searchValue}`); return cache.resultMap.get(searchValue); } // Clear cache if keys changed if (cache.keyHash !== currentKeyHash) { - console.log(`${type} Fuse keys changed, clearing cache`); + // console.debug(`${type} Fuse keys changed, clearing cache`); cache.keyHash = currentKeyHash; cache.resultMap.clear(); } @@ -1891,7 +1891,7 @@ function performFuzzySearch(type, data, keys, searchValue) { const results = fuse.search(searchValue); cache.resultMap.set(searchValue, results); - console.log(`${type} fuzzy search results for ${searchValue}`, results); + // console.debug(`${type} fuzzy search results for ${searchValue}`, results); return results; } From e1d6a47048e0d53921214607387f2687d2fb805f Mon Sep 17 00:00:00 2001 From: Joe Date: Sun, 24 Nov 2024 14:54:12 -0800 Subject: [PATCH 05/11] Remove key hashing and explicitly clear the cache after printing characters --- public/script.js | 2 ++ public/scripts/power-user.js | 27 +++++++++------------------ 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/public/script.js b/public/script.js index 57acf537e..4d6e41648 100644 --- a/public/script.js +++ b/public/script.js @@ -268,6 +268,7 @@ import { initServerHistory } from './scripts/server-history.js'; import { initSettingsSearch } from './scripts/setting-search.js'; import { initBulkEdit } from './scripts/bulk-edit.js'; import { deriveTemplatesFromChatTemplate } from './scripts/chat-templates.js'; +import { clearFuzzySearchCaches } from './scripts/power-user.js'; //exporting functions and vars for mods export { @@ -1515,6 +1516,7 @@ export async function printCharacters(fullRefresh = false) { }); favsToHotswap(); + clearFuzzySearchCaches(); } /** Checks the state of the current search, and adds/removes the search sorting option accordingly */ diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js index 8b9e837c2..91ba33844 100644 --- a/public/scripts/power-user.js +++ b/public/scripts/power-user.js @@ -329,11 +329,11 @@ let browser_has_focus = true; const debug_functions = []; const fuzzySearchCaches = { - characters: { keyHash: null, resultMap: new Map() }, - worldInfo: { keyHash: null, resultMap: new Map() }, - personas: { keyHash: null, resultMap: new Map() }, - tags: { keyHash: null, resultMap: new Map() }, - groups: { keyHash: null, resultMap: new Map() }, + characters: { resultMap: new Map() }, + worldInfo: { resultMap: new Map() }, + personas: { resultMap: new Map() }, + tags: { resultMap: new Map() }, + groups: { resultMap: new Map() }, }; const fuzzySearchCategories = { @@ -1864,22 +1864,14 @@ function hashFuseKeys(keys) { * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ function performFuzzySearch(type, data, keys, searchValue) { - const currentKeyHash = hashFuseKeys(keys); const cache = fuzzySearchCaches[type]; // Check cache for existing results - if (cache.keyHash === currentKeyHash && cache.resultMap.has(searchValue)) { - // console.debug(`Using cached ${type} fuzzy search results for ${searchValue}`); + if (cache.resultMap.has(searchValue)) { return cache.resultMap.get(searchValue); } - // Clear cache if keys changed - if (cache.keyHash !== currentKeyHash) { - // console.debug(`${type} Fuse keys changed, clearing cache`); - cache.keyHash = currentKeyHash; - cache.resultMap.clear(); - } - + // @ts-ignore const fuse = new Fuse(data, { keys: keys, includeScore: true, @@ -1890,19 +1882,18 @@ function performFuzzySearch(type, data, keys, searchValue) { const results = fuse.search(searchValue); cache.resultMap.set(searchValue, results); - - // console.debug(`${type} fuzzy search results for ${searchValue}`, results); return results; } + /** * Clears all fuzzy search caches */ export function clearFuzzySearchCaches() { for (const cache of Object.values(fuzzySearchCaches)) { - cache.keyHash = null; cache.resultMap.clear(); } + console.log('Fuzzy search caches cleared'); } /** From e56faaaed581626866c76b2bb154f76234ef797b Mon Sep 17 00:00:00 2001 From: Joe Date: Sun, 24 Nov 2024 14:55:22 -0800 Subject: [PATCH 06/11] Remove hash function --- public/scripts/power-user.js | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js index 91ba33844..5b780b700 100644 --- a/public/scripts/power-user.js +++ b/public/scripts/power-user.js @@ -1840,21 +1840,6 @@ async function loadContextSettings() { }); } -/** - * Generates a hash of the Fuse configuration keys - * @param {Array<{name: string, weight: number, getFn?: Function}>} keys - * @returns {number} Hash of the keys configuration - */ -function hashFuseKeys(keys) { - // Convert keys to a stable string representation - const keyString = keys.map(k => { - const getFnString = k.getFn ? k.getFn.toString() : ''; - return `${k.name}:${k.weight}:${getFnString}`; - }).join('|'); - - return getStringHash(keyString); -} - /** * Common function to perform fuzzy search with caching * @param {string} type - Type of search from fuzzySearchCategories From 16ba8331b5bd0bec42bf005551c51799a1e58c1e Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 26 Nov 2024 13:27:27 -0800 Subject: [PATCH 07/11] Document Fuzzy Search process --- docs/fuzzy_search_flow.md | 113 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 docs/fuzzy_search_flow.md diff --git a/docs/fuzzy_search_flow.md b/docs/fuzzy_search_flow.md new file mode 100644 index 000000000..b6d686c71 --- /dev/null +++ b/docs/fuzzy_search_flow.md @@ -0,0 +1,113 @@ +## Input Sources to printCharacters + + +`printCharacters` is the main function that triggers the fuzzy search process if fuzzy search is enabled. + +```mermaid +flowchart TD + subgraph User Actions + UA1[Character Search Input] + UA2[Tag Filter Click] + UA3[Folder Navigation] + UA4[Character Delete] + UA5[Character Create] + UA6[Character Import] + UA7[Clear All Filters] + UA8[Bulk Edit Operations] + UA9[Persona Changes] + end + + subgraph API Events + API1[Character List Update] + API2[Group Update] + API3[Tag Update] + end + + subgraph System Events + SE1[Page Load] + SE2[Content Manager Update] + SE3[Extension Events] + end + + UA1 -->|triggers| PCD[printCharactersDebounced] + UA2 -->|triggers| PCD + UA7 -->|triggers| PCD + UA8 -->|triggers| PCD + UA9 -->|triggers| PCD + + UA3 -->|triggers| PC[printCharacters] + UA4 -->|triggers| PC + UA5 -->|triggers| PC + UA6 -->|triggers| PC + + API1 -->|triggers| PC + API2 -->|triggers| PC + API3 -->|triggers| PC + + SE1 -->|triggers| PC + SE2 -->|triggers| PC + SE3 -->|triggers| PC + + PCD -->|debounced call| PC + + style PC fill:#f96,stroke:#333 + style PCD fill:#f96,stroke:#333 +``` + +This diagram shows how `printCharacters` is called throughout the application: + +1. User Actions that trigger character list updates: + - Search input (debounced) + - Tag filter clicks (debounced) + - Folder navigation (direct) + - Character management operations (direct) + +2. API Events that require list refresh: + - Character list updates + - Group updates + - Tag system updates + +3. System Events: + - Initial page load + - Content manager updates + - Extension-triggered refreshes + + + +## Fuzzy Search Flow + + +This diagram shows the flow of fuzzy search operations: +```mermaid +sequenceDiagram + participant Data as Data Sources + participant PC as printCharacters + participant GEL as getEntitiesList + participant FH as FilterHelper + participant AF as applyFilters + participant FS as FuzzySearch Functions + participant Cache as FuzzySearchCaches + + Note over Data: Changes from:
- Tags
- Personas
- World Info
- Groups + + Data->>PC: All changes trigger printCharacters
(direct or debounced) + + PC->>GEL: Call with {doFilter: true} + GEL->>FH: filterByTagState(entities) + GEL->>AF: entitiesFilter.applyFilters(entities) + + AF->>FH: Check scoreCache for existing results + FH-->>AF: Return cached scores if exist + + Note over FS: Filter functions include:
SEARCH,
FAV,
GROUP,
FOLDER,
TAG,
WORLD_INFO_SEARCH,
PERSONA_SEARCH + AF->>FS: fuzzySearchCharacters/Groups/Tags + FS->>Cache: Check/Store results + + FS-->>AF: Return search results + AF->>FH: Cache new scores + AF-->>GEL: Return filtered entities + GEL-->>PC: Return final entities list + + PC->>Cache: clearFuzzySearchCaches() + Note over Cache: Cache is cleared at the end of
each printCharacters call,
ensuring fresh results for next search +``` From dd4a1bf07227c8552a57c48fe0a762e51f0bfa63 Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 26 Nov 2024 13:32:40 -0800 Subject: [PATCH 08/11] Attempt to allow preview for github --- docs/fuzzy_search_flow.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/fuzzy_search_flow.md b/docs/fuzzy_search_flow.md index b6d686c71..af6af0e62 100644 --- a/docs/fuzzy_search_flow.md +++ b/docs/fuzzy_search_flow.md @@ -1,9 +1,9 @@ ## Input Sources to printCharacters - `printCharacters` is the main function that triggers the fuzzy search process if fuzzy search is enabled. ```mermaid +%%{init: {'flowchart': {'nodeSpacing': 50, 'rankSpacing': 50}}}%% flowchart TD subgraph User Actions UA1[Character Search Input] From 1395c0b8c6e4c1a59192051ecd72cd509dcdf709 Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 26 Nov 2024 19:47:09 -0800 Subject: [PATCH 09/11] Encapsulate logic into filters instead of spreading around --- docs/fuzzy_search_flow.md | 113 ----------------------------------- public/script.js | 10 ++-- public/scripts/filters.js | 57 ++++++++++++++++-- public/scripts/power-user.js | 66 +++++++++----------- 4 files changed, 84 insertions(+), 162 deletions(-) delete mode 100644 docs/fuzzy_search_flow.md diff --git a/docs/fuzzy_search_flow.md b/docs/fuzzy_search_flow.md deleted file mode 100644 index af6af0e62..000000000 --- a/docs/fuzzy_search_flow.md +++ /dev/null @@ -1,113 +0,0 @@ -## Input Sources to printCharacters - -`printCharacters` is the main function that triggers the fuzzy search process if fuzzy search is enabled. - -```mermaid -%%{init: {'flowchart': {'nodeSpacing': 50, 'rankSpacing': 50}}}%% -flowchart TD - subgraph User Actions - UA1[Character Search Input] - UA2[Tag Filter Click] - UA3[Folder Navigation] - UA4[Character Delete] - UA5[Character Create] - UA6[Character Import] - UA7[Clear All Filters] - UA8[Bulk Edit Operations] - UA9[Persona Changes] - end - - subgraph API Events - API1[Character List Update] - API2[Group Update] - API3[Tag Update] - end - - subgraph System Events - SE1[Page Load] - SE2[Content Manager Update] - SE3[Extension Events] - end - - UA1 -->|triggers| PCD[printCharactersDebounced] - UA2 -->|triggers| PCD - UA7 -->|triggers| PCD - UA8 -->|triggers| PCD - UA9 -->|triggers| PCD - - UA3 -->|triggers| PC[printCharacters] - UA4 -->|triggers| PC - UA5 -->|triggers| PC - UA6 -->|triggers| PC - - API1 -->|triggers| PC - API2 -->|triggers| PC - API3 -->|triggers| PC - - SE1 -->|triggers| PC - SE2 -->|triggers| PC - SE3 -->|triggers| PC - - PCD -->|debounced call| PC - - style PC fill:#f96,stroke:#333 - style PCD fill:#f96,stroke:#333 -``` - -This diagram shows how `printCharacters` is called throughout the application: - -1. User Actions that trigger character list updates: - - Search input (debounced) - - Tag filter clicks (debounced) - - Folder navigation (direct) - - Character management operations (direct) - -2. API Events that require list refresh: - - Character list updates - - Group updates - - Tag system updates - -3. System Events: - - Initial page load - - Content manager updates - - Extension-triggered refreshes - - - -## Fuzzy Search Flow - - -This diagram shows the flow of fuzzy search operations: -```mermaid -sequenceDiagram - participant Data as Data Sources - participant PC as printCharacters - participant GEL as getEntitiesList - participant FH as FilterHelper - participant AF as applyFilters - participant FS as FuzzySearch Functions - participant Cache as FuzzySearchCaches - - Note over Data: Changes from:
- Tags
- Personas
- World Info
- Groups - - Data->>PC: All changes trigger printCharacters
(direct or debounced) - - PC->>GEL: Call with {doFilter: true} - GEL->>FH: filterByTagState(entities) - GEL->>AF: entitiesFilter.applyFilters(entities) - - AF->>FH: Check scoreCache for existing results - FH-->>AF: Return cached scores if exist - - Note over FS: Filter functions include:
SEARCH,
FAV,
GROUP,
FOLDER,
TAG,
WORLD_INFO_SEARCH,
PERSONA_SEARCH - AF->>FS: fuzzySearchCharacters/Groups/Tags - FS->>Cache: Check/Store results - - FS-->>AF: Return search results - AF->>FH: Cache new scores - AF-->>GEL: Return filtered entities - GEL-->>PC: Return final entities list - - PC->>Cache: clearFuzzySearchCaches() - Note over Cache: Cache is cleared at the end of
each printCharacters call,
ensuring fresh results for next search -``` diff --git a/public/script.js b/public/script.js index 3db294dd5..22b5f9eac 100644 --- a/public/script.js +++ b/public/script.js @@ -268,7 +268,6 @@ import { initServerHistory } from './scripts/server-history.js'; import { initSettingsSearch } from './scripts/setting-search.js'; import { initBulkEdit } from './scripts/bulk-edit.js'; import { deriveTemplatesFromChatTemplate } from './scripts/chat-templates.js'; -import { clearFuzzySearchCaches } from './scripts/power-user.js'; //exporting functions and vars for mods export { @@ -1516,7 +1515,6 @@ export async function printCharacters(fullRefresh = false) { }); favsToHotswap(); - clearFuzzySearchCaches(); } /** Checks the state of the current search, and adds/removes the search sorting option accordingly */ @@ -1623,7 +1621,7 @@ export function getEntitiesList({ doFilter = false, doSort = true } = {}) { subEntities = filterByTagState(entities, { subForEntity: entity }); if (doFilter) { // sub entities filter "hacked" because folder filter should not be applied there, so even in "only folders" mode characters show up - subEntities = entitiesFilter.applyFilters(subEntities, { clearScoreCache: false, tempOverrides: { [FILTER_TYPES.FOLDER]: FILTER_STATES.UNDEFINED } }); + subEntities = entitiesFilter.applyFilters(subEntities, { clearScoreCache: false, tempOverrides: { [FILTER_TYPES.FOLDER]: FILTER_STATES.UNDEFINED }, clearFuzzySearchCaches: false }); } if (doSort) { sortEntitiesList(subEntities); @@ -1636,11 +1634,11 @@ export function getEntitiesList({ doFilter = false, doSort = true } = {}) { // Second run filters, hiding whatever should be filtered later if (doFilter) { const beforeFinalEntities = filterByTagState(entities, { globalDisplayFilters: true }); - entities = entitiesFilter.applyFilters(beforeFinalEntities); + entities = entitiesFilter.applyFilters(beforeFinalEntities, { clearFuzzySearchCaches: false }); // Magic for folder filter. If that one is enabled, and no folders are display anymore, we remove that filter to actually show the characters. if (isFilterState(entitiesFilter.getFilterData(FILTER_TYPES.FOLDER), FILTER_STATES.SELECTED) && entities.filter(x => x.type == 'tag').length == 0) { - entities = entitiesFilter.applyFilters(beforeFinalEntities, { tempOverrides: { [FILTER_TYPES.FOLDER]: FILTER_STATES.UNDEFINED } }); + entities = entitiesFilter.applyFilters(beforeFinalEntities, { tempOverrides: { [FILTER_TYPES.FOLDER]: FILTER_STATES.UNDEFINED }, clearFuzzySearchCaches: false }); } } @@ -1656,6 +1654,7 @@ export function getEntitiesList({ doFilter = false, doSort = true } = {}) { if (doSort) { sortEntitiesList(entities); } + entitiesFilter.clearFuzzySearchCaches(); return entities; } @@ -1751,6 +1750,7 @@ export async function getCharacters() { } await getGroups(); + // clearFuzzySearchCaches(); await printCharacters(true); } } diff --git a/public/scripts/filters.js b/public/scripts/filters.js index adc27b349..2d01e45e3 100644 --- a/public/scripts/filters.js +++ b/public/scripts/filters.js @@ -55,6 +55,19 @@ export function isFilterState(a, b) { return aKey === bKey; } +/** + * The fuzzy search categories + * @type {{ characters: string, worldInfo: string, personas: string, tags: string, groups: string }} + */ +export const fuzzySearchCategories = Object.freeze({ + characters: 'characters', + worldInfo: 'worldInfo', + personas: 'personas', + tags: 'tags', + groups: 'groups', +}); + + /** * Helper class for filtering data. * @example @@ -72,6 +85,12 @@ export class FilterHelper { */ scoreCache; + /** + * Cache for fuzzy search results per category. + * @type {Object. }>} + */ + fuzzySearchCaches; + /** * Creates a new FilterHelper * @param {Function} onDataChanged Callback to trigger when the filter data changes @@ -79,6 +98,13 @@ export class FilterHelper { constructor(onDataChanged) { this.onDataChanged = onDataChanged; this.scoreCache = new Map(); + this.fuzzySearchCaches = { + [fuzzySearchCategories.characters]: { resultMap: new Map() }, + [fuzzySearchCategories.worldInfo]: { resultMap: new Map() }, + [fuzzySearchCategories.personas]: { resultMap: new Map() }, + [fuzzySearchCategories.tags]: { resultMap: new Map() }, + [fuzzySearchCategories.groups]: { resultMap: new Map() }, + }; } /** @@ -151,7 +177,7 @@ export class FilterHelper { return data; } - const fuzzySearchResults = fuzzySearchWorldInfo(data, term); + const fuzzySearchResults = fuzzySearchWorldInfo(data, term, this.fuzzySearchCaches); this.cacheScores(FILTER_TYPES.WORLD_INFO_SEARCH, new Map(fuzzySearchResults.map(i => [i.item?.uid, i.score]))); const filteredData = data.filter(entity => fuzzySearchResults.find(x => x.item === entity)); @@ -170,7 +196,7 @@ export class FilterHelper { return data; } - const fuzzySearchResults = fuzzySearchPersonas(data, term); + const fuzzySearchResults = fuzzySearchPersonas(data, term, this.fuzzySearchCaches); this.cacheScores(FILTER_TYPES.PERSONA_SEARCH, new Map(fuzzySearchResults.map(i => [i.item.key, i.score]))); const filteredData = data.filter(name => fuzzySearchResults.find(x => x.item.key === name)); @@ -289,9 +315,9 @@ export class FilterHelper { // Save fuzzy search results and scores if enabled if (power_user.fuzzy_search) { - const fuzzySearchCharactersResults = fuzzySearchCharacters(searchValue); - const fuzzySearchGroupsResults = fuzzySearchGroups(searchValue); - const fuzzySearchTagsResult = fuzzySearchTags(searchValue); + const fuzzySearchCharactersResults = fuzzySearchCharacters(searchValue, this.fuzzySearchCaches); + const fuzzySearchGroupsResults = fuzzySearchGroups(searchValue, this.fuzzySearchCaches); + const fuzzySearchTagsResult = fuzzySearchTags(searchValue, this.fuzzySearchCaches); this.cacheScores(FILTER_TYPES.SEARCH, new Map(fuzzySearchCharactersResults.map(i => [`character.${i.refIndex}`, i.score]))); this.cacheScores(FILTER_TYPES.SEARCH, new Map(fuzzySearchGroupsResults.map(i => [`group.${i.item.id}`, i.score]))); this.cacheScores(FILTER_TYPES.SEARCH, new Map(fuzzySearchTagsResult.map(i => [`tag.${i.item.id}`, i.score]))); @@ -343,11 +369,14 @@ export class FilterHelper { * @param {object} options - Optional call parameters * @param {boolean} [options.clearScoreCache=true] - Whether the score cache should be cleared. * @param {Object.} [options.tempOverrides={}] - Temporarily override specific filters for this filter application + * @param {boolean} [options.clearFuzzySearchCaches=true] - Whether the fuzzy search caches should be cleared. * @returns {any[]} The filtered data. */ - applyFilters(data, { clearScoreCache = true, tempOverrides = {} } = {}) { + applyFilters(data, { clearScoreCache = true, tempOverrides = {}, clearFuzzySearchCaches = true } = {}) { if (clearScoreCache) this.clearScoreCache(); + if (clearFuzzySearchCaches) this.clearFuzzySearchCaches(); + // Save original filter states const originalStates = {}; for (const key in tempOverrides) { @@ -411,4 +440,20 @@ export class FilterHelper { this.scoreCache = new Map(); } } + + /** + * Clears fuzzy search caches + * @param {keyof typeof fuzzySearchCategories} [type] Optional cache type to clear. If not provided, clears all caches + */ + clearFuzzySearchCaches(type = null) { + if (type && this.fuzzySearchCaches[type]) { + this.fuzzySearchCaches[type].resultMap.clear(); + console.log(`Fuzzy search cache cleared for: ${type}`); + } else { + for (const cache of Object.values(this.fuzzySearchCaches)) { + cache.resultMap.clear(); + } + console.log('All fuzzy search caches cleared'); + } + } } diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js index 5b64fcee8..0a36952f2 100644 --- a/public/scripts/power-user.js +++ b/public/scripts/power-user.js @@ -53,6 +53,7 @@ import { SlashCommandEnumValue, enumTypes } from './slash-commands/SlashCommandE import { commonEnumProviders, enumIcons } from './slash-commands/SlashCommandCommonEnumsProvider.js'; import { POPUP_TYPE, callGenericPopup } from './popup.js'; import { loadSystemPrompts } from './sysprompt.js'; +import { fuzzySearchCategories } from './filters.js'; export { loadPowerUserSettings, @@ -328,22 +329,6 @@ const contextControls = [ let browser_has_focus = true; const debug_functions = []; -const fuzzySearchCaches = { - characters: { resultMap: new Map() }, - worldInfo: { resultMap: new Map() }, - personas: { resultMap: new Map() }, - tags: { resultMap: new Map() }, - groups: { resultMap: new Map() }, -}; - -const fuzzySearchCategories = { - characters: 'characters', - worldInfo: 'worldInfo', - personas: 'personas', - tags: 'tags', - groups: 'groups', -}; - const setHotswapsDebounced = debounce(favsToHotswap); function playMessageSound() { @@ -1845,19 +1830,26 @@ async function loadContextSettings() { }); } + /** * Common function to perform fuzzy search with caching * @param {string} type - Type of search from fuzzySearchCategories * @param {any[]} data - Data array to search in * @param {Array<{name: string, weight: number, getFn?: Function}>} keys - Fuse.js keys configuration * @param {string} searchValue - The search term + * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -function performFuzzySearch(type, data, keys, searchValue) { +function performFuzzySearch(type, data, keys, searchValue, fuzzySearchCaches) { + let startTime = performance.now(); const cache = fuzzySearchCaches[type]; // Check cache for existing results if (cache.resultMap.has(searchValue)) { + let endTime = performance.now(); + if (endTime - startTime > 1.0) { + console.log(`Fuzzy search for ${type} took ${endTime - startTime}ms (cached)`); + } return cache.resultMap.get(searchValue); } @@ -1872,26 +1864,20 @@ function performFuzzySearch(type, data, keys, searchValue) { const results = fuse.search(searchValue); cache.resultMap.set(searchValue, results); - return results; -} - - -/** - * Clears all fuzzy search caches - */ -export function clearFuzzySearchCaches() { - for (const cache of Object.values(fuzzySearchCaches)) { - cache.resultMap.clear(); + let endTime = performance.now(); + if (endTime - startTime > 1.0) { + console.log(`Fuzzy search for ${type} took ${endTime - startTime}ms`); } - console.log('Fuzzy search caches cleared'); + return results; } /** * Fuzzy search characters by a search term * @param {string} searchValue - The search term + * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -export function fuzzySearchCharacters(searchValue) { +export function fuzzySearchCharacters(searchValue, fuzzySearchCaches) { const keys = [ { name: 'data.name', weight: 20 }, { name: '#tags', weight: 10, getFn: (character) => getTagsList(character.avatar).map(x => x.name).join('||') }, @@ -1906,16 +1892,17 @@ export function fuzzySearchCharacters(searchValue) { { name: 'data.alternate_greetings', weight: 1 }, ]; - return performFuzzySearch(fuzzySearchCategories.characters, characters, keys, searchValue); + return performFuzzySearch(fuzzySearchCategories.characters, characters, keys, searchValue, fuzzySearchCaches); } /** * Fuzzy search world info entries by a search term * @param {*[]} data - WI items data array * @param {string} searchValue - The search term + * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -export function fuzzySearchWorldInfo(data, searchValue) { +export function fuzzySearchWorldInfo(data, searchValue, fuzzySearchCaches) { const keys = [ { name: 'key', weight: 20 }, { name: 'group', weight: 15 }, @@ -1926,16 +1913,17 @@ export function fuzzySearchWorldInfo(data, searchValue) { { name: 'automationId', weight: 1 }, ]; - return performFuzzySearch(fuzzySearchCategories.worldInfo, data, keys, searchValue); + return performFuzzySearch(fuzzySearchCategories.worldInfo, data, keys, searchValue, fuzzySearchCaches); } /** * Fuzzy search persona entries by a search term * @param {*[]} data - persona data array * @param {string} searchValue - The search term + * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -export function fuzzySearchPersonas(data, searchValue) { +export function fuzzySearchPersonas(data, searchValue, fuzzySearchCaches) { const mappedData = data.map(x => ({ key: x, name: power_user.personas[x] ?? '', @@ -1947,28 +1935,30 @@ export function fuzzySearchPersonas(data, searchValue) { { name: 'description', weight: 3 }, ]; - return performFuzzySearch(fuzzySearchCategories.personas, mappedData, keys, searchValue); + return performFuzzySearch(fuzzySearchCategories.personas, mappedData, keys, searchValue, fuzzySearchCaches); } /** * Fuzzy search tags by a search term * @param {string} searchValue - The search term + * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -export function fuzzySearchTags(searchValue) { +export function fuzzySearchTags(searchValue, fuzzySearchCaches) { const keys = [ { name: 'name', weight: 1 }, ]; - return performFuzzySearch(fuzzySearchCategories.tags, tags, keys, searchValue); + return performFuzzySearch(fuzzySearchCategories.tags, tags, keys, searchValue, fuzzySearchCaches); } /** * Fuzzy search groups by a search term * @param {string} searchValue - The search term + * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -export function fuzzySearchGroups(searchValue) { +export function fuzzySearchGroups(searchValue, fuzzySearchCaches) { const keys = [ { name: 'name', weight: 20 }, { name: 'members', weight: 15 }, @@ -1976,7 +1966,7 @@ export function fuzzySearchGroups(searchValue) { { name: 'id', weight: 1 }, ]; - return performFuzzySearch(fuzzySearchCategories.groups, groups, keys, searchValue); + return performFuzzySearch(fuzzySearchCategories.groups, groups, keys, searchValue, fuzzySearchCaches); } /** From 309157dd8932518d5f1cc4349e022bdbf832ec66 Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 26 Nov 2024 19:52:21 -0800 Subject: [PATCH 10/11] Remove unnecesary key only clearing for cache clearing --- public/scripts/filters.js | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/public/scripts/filters.js b/public/scripts/filters.js index 2d01e45e3..f07b4e33f 100644 --- a/public/scripts/filters.js +++ b/public/scripts/filters.js @@ -443,17 +443,11 @@ export class FilterHelper { /** * Clears fuzzy search caches - * @param {keyof typeof fuzzySearchCategories} [type] Optional cache type to clear. If not provided, clears all caches */ - clearFuzzySearchCaches(type = null) { - if (type && this.fuzzySearchCaches[type]) { - this.fuzzySearchCaches[type].resultMap.clear(); - console.log(`Fuzzy search cache cleared for: ${type}`); - } else { - for (const cache of Object.values(this.fuzzySearchCaches)) { - cache.resultMap.clear(); - } - console.log('All fuzzy search caches cleared'); + clearFuzzySearchCaches() { + for (const cache of Object.values(this.fuzzySearchCaches)) { + cache.resultMap.clear(); } + console.log('All fuzzy search caches cleared'); } } From f4ef9697e983a0bb0b3b0ab6388f5ecbfe37898e Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 26 Nov 2024 20:07:30 -0800 Subject: [PATCH 11/11] Account for optional cache and remove timing code --- public/script.js | 1 - public/scripts/power-user.js | 48 +++++++++++++++++------------------- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/public/script.js b/public/script.js index 22b5f9eac..4d97e91b9 100644 --- a/public/script.js +++ b/public/script.js @@ -1750,7 +1750,6 @@ export async function getCharacters() { } await getGroups(); - // clearFuzzySearchCaches(); await printCharacters(true); } } diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js index 0a36952f2..931067333 100644 --- a/public/scripts/power-user.js +++ b/public/scripts/power-user.js @@ -1832,25 +1832,21 @@ async function loadContextSettings() { /** - * Common function to perform fuzzy search with caching + * Common function to perform fuzzy search with optional caching * @param {string} type - Type of search from fuzzySearchCategories * @param {any[]} data - Data array to search in * @param {Array<{name: string, weight: number, getFn?: Function}>} keys - Fuse.js keys configuration * @param {string} searchValue - The search term - * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches + * @param {Object. }>} [fuzzySearchCaches=null] - Optional fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -function performFuzzySearch(type, data, keys, searchValue, fuzzySearchCaches) { - let startTime = performance.now(); - const cache = fuzzySearchCaches[type]; - - // Check cache for existing results - if (cache.resultMap.has(searchValue)) { - let endTime = performance.now(); - if (endTime - startTime > 1.0) { - console.log(`Fuzzy search for ${type} took ${endTime - startTime}ms (cached)`); +function performFuzzySearch(type, data, keys, searchValue, fuzzySearchCaches = null) { + // Check cache if provided + if (fuzzySearchCaches) { + const cache = fuzzySearchCaches[type]; + if (cache?.resultMap.has(searchValue)) { + return cache.resultMap.get(searchValue); } - return cache.resultMap.get(searchValue); } // @ts-ignore @@ -1863,10 +1859,10 @@ function performFuzzySearch(type, data, keys, searchValue, fuzzySearchCaches) { }); const results = fuse.search(searchValue); - cache.resultMap.set(searchValue, results); - let endTime = performance.now(); - if (endTime - startTime > 1.0) { - console.log(`Fuzzy search for ${type} took ${endTime - startTime}ms`); + + // Store in cache if provided + if (fuzzySearchCaches) { + fuzzySearchCaches[type].resultMap.set(searchValue, results); } return results; } @@ -1874,10 +1870,10 @@ function performFuzzySearch(type, data, keys, searchValue, fuzzySearchCaches) { /** * Fuzzy search characters by a search term * @param {string} searchValue - The search term - * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches + * @param {Object. }>} [fuzzySearchCaches=null] - Optional fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -export function fuzzySearchCharacters(searchValue, fuzzySearchCaches) { +export function fuzzySearchCharacters(searchValue, fuzzySearchCaches = null) { const keys = [ { name: 'data.name', weight: 20 }, { name: '#tags', weight: 10, getFn: (character) => getTagsList(character.avatar).map(x => x.name).join('||') }, @@ -1899,10 +1895,10 @@ export function fuzzySearchCharacters(searchValue, fuzzySearchCaches) { * Fuzzy search world info entries by a search term * @param {*[]} data - WI items data array * @param {string} searchValue - The search term - * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches + * @param {Object. }>} [fuzzySearchCaches=null] - Optional fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -export function fuzzySearchWorldInfo(data, searchValue, fuzzySearchCaches) { +export function fuzzySearchWorldInfo(data, searchValue, fuzzySearchCaches = null) { const keys = [ { name: 'key', weight: 20 }, { name: 'group', weight: 15 }, @@ -1920,10 +1916,10 @@ export function fuzzySearchWorldInfo(data, searchValue, fuzzySearchCaches) { * Fuzzy search persona entries by a search term * @param {*[]} data - persona data array * @param {string} searchValue - The search term - * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches + * @param {Object. }>} [fuzzySearchCaches=null] - Optional fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -export function fuzzySearchPersonas(data, searchValue, fuzzySearchCaches) { +export function fuzzySearchPersonas(data, searchValue, fuzzySearchCaches = null) { const mappedData = data.map(x => ({ key: x, name: power_user.personas[x] ?? '', @@ -1941,10 +1937,10 @@ export function fuzzySearchPersonas(data, searchValue, fuzzySearchCaches) { /** * Fuzzy search tags by a search term * @param {string} searchValue - The search term - * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches + * @param {Object. }>} [fuzzySearchCaches=null] - Optional fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -export function fuzzySearchTags(searchValue, fuzzySearchCaches) { +export function fuzzySearchTags(searchValue, fuzzySearchCaches = null) { const keys = [ { name: 'name', weight: 1 }, ]; @@ -1955,10 +1951,10 @@ export function fuzzySearchTags(searchValue, fuzzySearchCaches) { /** * Fuzzy search groups by a search term * @param {string} searchValue - The search term - * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches + * @param {Object. }>} [fuzzySearchCaches=null] - Optional fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -export function fuzzySearchGroups(searchValue, fuzzySearchCaches) { +export function fuzzySearchGroups(searchValue, fuzzySearchCaches = null) { const keys = [ { name: 'name', weight: 20 }, { name: 'members', weight: 15 },