diff --git a/docs/fuzzy_search_flow.md b/docs/fuzzy_search_flow.md deleted file mode 100644 index af6af0e62..000000000 --- a/docs/fuzzy_search_flow.md +++ /dev/null @@ -1,113 +0,0 @@ -## Input Sources to printCharacters - -`printCharacters` is the main function that triggers the fuzzy search process if fuzzy search is enabled. - -```mermaid -%%{init: {'flowchart': {'nodeSpacing': 50, 'rankSpacing': 50}}}%% -flowchart TD - subgraph User Actions - UA1[Character Search Input] - UA2[Tag Filter Click] - UA3[Folder Navigation] - UA4[Character Delete] - UA5[Character Create] - UA6[Character Import] - UA7[Clear All Filters] - UA8[Bulk Edit Operations] - UA9[Persona Changes] - end - - subgraph API Events - API1[Character List Update] - API2[Group Update] - API3[Tag Update] - end - - subgraph System Events - SE1[Page Load] - SE2[Content Manager Update] - SE3[Extension Events] - end - - UA1 -->|triggers| PCD[printCharactersDebounced] - UA2 -->|triggers| PCD - UA7 -->|triggers| PCD - UA8 -->|triggers| PCD - UA9 -->|triggers| PCD - - UA3 -->|triggers| PC[printCharacters] - UA4 -->|triggers| PC - UA5 -->|triggers| PC - UA6 -->|triggers| PC - - API1 -->|triggers| PC - API2 -->|triggers| PC - API3 -->|triggers| PC - - SE1 -->|triggers| PC - SE2 -->|triggers| PC - SE3 -->|triggers| PC - - PCD -->|debounced call| PC - - style PC fill:#f96,stroke:#333 - style PCD fill:#f96,stroke:#333 -``` - -This diagram shows how `printCharacters` is called throughout the application: - -1. User Actions that trigger character list updates: - - Search input (debounced) - - Tag filter clicks (debounced) - - Folder navigation (direct) - - Character management operations (direct) - -2. API Events that require list refresh: - - Character list updates - - Group updates - - Tag system updates - -3. System Events: - - Initial page load - - Content manager updates - - Extension-triggered refreshes - - - -## Fuzzy Search Flow - - -This diagram shows the flow of fuzzy search operations: -```mermaid -sequenceDiagram - participant Data as Data Sources - participant PC as printCharacters - participant GEL as getEntitiesList - participant FH as FilterHelper - participant AF as applyFilters - participant FS as FuzzySearch Functions - participant Cache as FuzzySearchCaches - - Note over Data: Changes from:
- Tags
- Personas
- World Info
- Groups - - Data->>PC: All changes trigger printCharacters
(direct or debounced) - - PC->>GEL: Call with {doFilter: true} - GEL->>FH: filterByTagState(entities) - GEL->>AF: entitiesFilter.applyFilters(entities) - - AF->>FH: Check scoreCache for existing results - FH-->>AF: Return cached scores if exist - - Note over FS: Filter functions include:
SEARCH,
FAV,
GROUP,
FOLDER,
TAG,
WORLD_INFO_SEARCH,
PERSONA_SEARCH - AF->>FS: fuzzySearchCharacters/Groups/Tags - FS->>Cache: Check/Store results - - FS-->>AF: Return search results - AF->>FH: Cache new scores - AF-->>GEL: Return filtered entities - GEL-->>PC: Return final entities list - - PC->>Cache: clearFuzzySearchCaches() - Note over Cache: Cache is cleared at the end of
each printCharacters call,
ensuring fresh results for next search -``` diff --git a/public/script.js b/public/script.js index 3db294dd5..22b5f9eac 100644 --- a/public/script.js +++ b/public/script.js @@ -268,7 +268,6 @@ import { initServerHistory } from './scripts/server-history.js'; import { initSettingsSearch } from './scripts/setting-search.js'; import { initBulkEdit } from './scripts/bulk-edit.js'; import { deriveTemplatesFromChatTemplate } from './scripts/chat-templates.js'; -import { clearFuzzySearchCaches } from './scripts/power-user.js'; //exporting functions and vars for mods export { @@ -1516,7 +1515,6 @@ export async function printCharacters(fullRefresh = false) { }); favsToHotswap(); - clearFuzzySearchCaches(); } /** Checks the state of the current search, and adds/removes the search sorting option accordingly */ @@ -1623,7 +1621,7 @@ export function getEntitiesList({ doFilter = false, doSort = true } = {}) { subEntities = filterByTagState(entities, { subForEntity: entity }); if (doFilter) { // sub entities filter "hacked" because folder filter should not be applied there, so even in "only folders" mode characters show up - subEntities = entitiesFilter.applyFilters(subEntities, { clearScoreCache: false, tempOverrides: { [FILTER_TYPES.FOLDER]: FILTER_STATES.UNDEFINED } }); + subEntities = entitiesFilter.applyFilters(subEntities, { clearScoreCache: false, tempOverrides: { [FILTER_TYPES.FOLDER]: FILTER_STATES.UNDEFINED }, clearFuzzySearchCaches: false }); } if (doSort) { sortEntitiesList(subEntities); @@ -1636,11 +1634,11 @@ export function getEntitiesList({ doFilter = false, doSort = true } = {}) { // Second run filters, hiding whatever should be filtered later if (doFilter) { const beforeFinalEntities = filterByTagState(entities, { globalDisplayFilters: true }); - entities = entitiesFilter.applyFilters(beforeFinalEntities); + entities = entitiesFilter.applyFilters(beforeFinalEntities, { clearFuzzySearchCaches: false }); // Magic for folder filter. If that one is enabled, and no folders are display anymore, we remove that filter to actually show the characters. if (isFilterState(entitiesFilter.getFilterData(FILTER_TYPES.FOLDER), FILTER_STATES.SELECTED) && entities.filter(x => x.type == 'tag').length == 0) { - entities = entitiesFilter.applyFilters(beforeFinalEntities, { tempOverrides: { [FILTER_TYPES.FOLDER]: FILTER_STATES.UNDEFINED } }); + entities = entitiesFilter.applyFilters(beforeFinalEntities, { tempOverrides: { [FILTER_TYPES.FOLDER]: FILTER_STATES.UNDEFINED }, clearFuzzySearchCaches: false }); } } @@ -1656,6 +1654,7 @@ export function getEntitiesList({ doFilter = false, doSort = true } = {}) { if (doSort) { sortEntitiesList(entities); } + entitiesFilter.clearFuzzySearchCaches(); return entities; } @@ -1751,6 +1750,7 @@ export async function getCharacters() { } await getGroups(); + // clearFuzzySearchCaches(); await printCharacters(true); } } diff --git a/public/scripts/filters.js b/public/scripts/filters.js index adc27b349..2d01e45e3 100644 --- a/public/scripts/filters.js +++ b/public/scripts/filters.js @@ -55,6 +55,19 @@ export function isFilterState(a, b) { return aKey === bKey; } +/** + * The fuzzy search categories + * @type {{ characters: string, worldInfo: string, personas: string, tags: string, groups: string }} + */ +export const fuzzySearchCategories = Object.freeze({ + characters: 'characters', + worldInfo: 'worldInfo', + personas: 'personas', + tags: 'tags', + groups: 'groups', +}); + + /** * Helper class for filtering data. * @example @@ -72,6 +85,12 @@ export class FilterHelper { */ scoreCache; + /** + * Cache for fuzzy search results per category. + * @type {Object. }>} + */ + fuzzySearchCaches; + /** * Creates a new FilterHelper * @param {Function} onDataChanged Callback to trigger when the filter data changes @@ -79,6 +98,13 @@ export class FilterHelper { constructor(onDataChanged) { this.onDataChanged = onDataChanged; this.scoreCache = new Map(); + this.fuzzySearchCaches = { + [fuzzySearchCategories.characters]: { resultMap: new Map() }, + [fuzzySearchCategories.worldInfo]: { resultMap: new Map() }, + [fuzzySearchCategories.personas]: { resultMap: new Map() }, + [fuzzySearchCategories.tags]: { resultMap: new Map() }, + [fuzzySearchCategories.groups]: { resultMap: new Map() }, + }; } /** @@ -151,7 +177,7 @@ export class FilterHelper { return data; } - const fuzzySearchResults = fuzzySearchWorldInfo(data, term); + const fuzzySearchResults = fuzzySearchWorldInfo(data, term, this.fuzzySearchCaches); this.cacheScores(FILTER_TYPES.WORLD_INFO_SEARCH, new Map(fuzzySearchResults.map(i => [i.item?.uid, i.score]))); const filteredData = data.filter(entity => fuzzySearchResults.find(x => x.item === entity)); @@ -170,7 +196,7 @@ export class FilterHelper { return data; } - const fuzzySearchResults = fuzzySearchPersonas(data, term); + const fuzzySearchResults = fuzzySearchPersonas(data, term, this.fuzzySearchCaches); this.cacheScores(FILTER_TYPES.PERSONA_SEARCH, new Map(fuzzySearchResults.map(i => [i.item.key, i.score]))); const filteredData = data.filter(name => fuzzySearchResults.find(x => x.item.key === name)); @@ -289,9 +315,9 @@ export class FilterHelper { // Save fuzzy search results and scores if enabled if (power_user.fuzzy_search) { - const fuzzySearchCharactersResults = fuzzySearchCharacters(searchValue); - const fuzzySearchGroupsResults = fuzzySearchGroups(searchValue); - const fuzzySearchTagsResult = fuzzySearchTags(searchValue); + const fuzzySearchCharactersResults = fuzzySearchCharacters(searchValue, this.fuzzySearchCaches); + const fuzzySearchGroupsResults = fuzzySearchGroups(searchValue, this.fuzzySearchCaches); + const fuzzySearchTagsResult = fuzzySearchTags(searchValue, this.fuzzySearchCaches); this.cacheScores(FILTER_TYPES.SEARCH, new Map(fuzzySearchCharactersResults.map(i => [`character.${i.refIndex}`, i.score]))); this.cacheScores(FILTER_TYPES.SEARCH, new Map(fuzzySearchGroupsResults.map(i => [`group.${i.item.id}`, i.score]))); this.cacheScores(FILTER_TYPES.SEARCH, new Map(fuzzySearchTagsResult.map(i => [`tag.${i.item.id}`, i.score]))); @@ -343,11 +369,14 @@ export class FilterHelper { * @param {object} options - Optional call parameters * @param {boolean} [options.clearScoreCache=true] - Whether the score cache should be cleared. * @param {Object.} [options.tempOverrides={}] - Temporarily override specific filters for this filter application + * @param {boolean} [options.clearFuzzySearchCaches=true] - Whether the fuzzy search caches should be cleared. * @returns {any[]} The filtered data. */ - applyFilters(data, { clearScoreCache = true, tempOverrides = {} } = {}) { + applyFilters(data, { clearScoreCache = true, tempOverrides = {}, clearFuzzySearchCaches = true } = {}) { if (clearScoreCache) this.clearScoreCache(); + if (clearFuzzySearchCaches) this.clearFuzzySearchCaches(); + // Save original filter states const originalStates = {}; for (const key in tempOverrides) { @@ -411,4 +440,20 @@ export class FilterHelper { this.scoreCache = new Map(); } } + + /** + * Clears fuzzy search caches + * @param {keyof typeof fuzzySearchCategories} [type] Optional cache type to clear. If not provided, clears all caches + */ + clearFuzzySearchCaches(type = null) { + if (type && this.fuzzySearchCaches[type]) { + this.fuzzySearchCaches[type].resultMap.clear(); + console.log(`Fuzzy search cache cleared for: ${type}`); + } else { + for (const cache of Object.values(this.fuzzySearchCaches)) { + cache.resultMap.clear(); + } + console.log('All fuzzy search caches cleared'); + } + } } diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js index 5b64fcee8..0a36952f2 100644 --- a/public/scripts/power-user.js +++ b/public/scripts/power-user.js @@ -53,6 +53,7 @@ import { SlashCommandEnumValue, enumTypes } from './slash-commands/SlashCommandE import { commonEnumProviders, enumIcons } from './slash-commands/SlashCommandCommonEnumsProvider.js'; import { POPUP_TYPE, callGenericPopup } from './popup.js'; import { loadSystemPrompts } from './sysprompt.js'; +import { fuzzySearchCategories } from './filters.js'; export { loadPowerUserSettings, @@ -328,22 +329,6 @@ const contextControls = [ let browser_has_focus = true; const debug_functions = []; -const fuzzySearchCaches = { - characters: { resultMap: new Map() }, - worldInfo: { resultMap: new Map() }, - personas: { resultMap: new Map() }, - tags: { resultMap: new Map() }, - groups: { resultMap: new Map() }, -}; - -const fuzzySearchCategories = { - characters: 'characters', - worldInfo: 'worldInfo', - personas: 'personas', - tags: 'tags', - groups: 'groups', -}; - const setHotswapsDebounced = debounce(favsToHotswap); function playMessageSound() { @@ -1845,19 +1830,26 @@ async function loadContextSettings() { }); } + /** * Common function to perform fuzzy search with caching * @param {string} type - Type of search from fuzzySearchCategories * @param {any[]} data - Data array to search in * @param {Array<{name: string, weight: number, getFn?: Function}>} keys - Fuse.js keys configuration * @param {string} searchValue - The search term + * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -function performFuzzySearch(type, data, keys, searchValue) { +function performFuzzySearch(type, data, keys, searchValue, fuzzySearchCaches) { + let startTime = performance.now(); const cache = fuzzySearchCaches[type]; // Check cache for existing results if (cache.resultMap.has(searchValue)) { + let endTime = performance.now(); + if (endTime - startTime > 1.0) { + console.log(`Fuzzy search for ${type} took ${endTime - startTime}ms (cached)`); + } return cache.resultMap.get(searchValue); } @@ -1872,26 +1864,20 @@ function performFuzzySearch(type, data, keys, searchValue) { const results = fuse.search(searchValue); cache.resultMap.set(searchValue, results); - return results; -} - - -/** - * Clears all fuzzy search caches - */ -export function clearFuzzySearchCaches() { - for (const cache of Object.values(fuzzySearchCaches)) { - cache.resultMap.clear(); + let endTime = performance.now(); + if (endTime - startTime > 1.0) { + console.log(`Fuzzy search for ${type} took ${endTime - startTime}ms`); } - console.log('Fuzzy search caches cleared'); + return results; } /** * Fuzzy search characters by a search term * @param {string} searchValue - The search term + * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -export function fuzzySearchCharacters(searchValue) { +export function fuzzySearchCharacters(searchValue, fuzzySearchCaches) { const keys = [ { name: 'data.name', weight: 20 }, { name: '#tags', weight: 10, getFn: (character) => getTagsList(character.avatar).map(x => x.name).join('||') }, @@ -1906,16 +1892,17 @@ export function fuzzySearchCharacters(searchValue) { { name: 'data.alternate_greetings', weight: 1 }, ]; - return performFuzzySearch(fuzzySearchCategories.characters, characters, keys, searchValue); + return performFuzzySearch(fuzzySearchCategories.characters, characters, keys, searchValue, fuzzySearchCaches); } /** * Fuzzy search world info entries by a search term * @param {*[]} data - WI items data array * @param {string} searchValue - The search term + * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -export function fuzzySearchWorldInfo(data, searchValue) { +export function fuzzySearchWorldInfo(data, searchValue, fuzzySearchCaches) { const keys = [ { name: 'key', weight: 20 }, { name: 'group', weight: 15 }, @@ -1926,16 +1913,17 @@ export function fuzzySearchWorldInfo(data, searchValue) { { name: 'automationId', weight: 1 }, ]; - return performFuzzySearch(fuzzySearchCategories.worldInfo, data, keys, searchValue); + return performFuzzySearch(fuzzySearchCategories.worldInfo, data, keys, searchValue, fuzzySearchCaches); } /** * Fuzzy search persona entries by a search term * @param {*[]} data - persona data array * @param {string} searchValue - The search term + * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -export function fuzzySearchPersonas(data, searchValue) { +export function fuzzySearchPersonas(data, searchValue, fuzzySearchCaches) { const mappedData = data.map(x => ({ key: x, name: power_user.personas[x] ?? '', @@ -1947,28 +1935,30 @@ export function fuzzySearchPersonas(data, searchValue) { { name: 'description', weight: 3 }, ]; - return performFuzzySearch(fuzzySearchCategories.personas, mappedData, keys, searchValue); + return performFuzzySearch(fuzzySearchCategories.personas, mappedData, keys, searchValue, fuzzySearchCaches); } /** * Fuzzy search tags by a search term * @param {string} searchValue - The search term + * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -export function fuzzySearchTags(searchValue) { +export function fuzzySearchTags(searchValue, fuzzySearchCaches) { const keys = [ { name: 'name', weight: 1 }, ]; - return performFuzzySearch(fuzzySearchCategories.tags, tags, keys, searchValue); + return performFuzzySearch(fuzzySearchCategories.tags, tags, keys, searchValue, fuzzySearchCaches); } /** * Fuzzy search groups by a search term * @param {string} searchValue - The search term + * @param {Object. }>} fuzzySearchCaches - Fuzzy search caches * @returns {import('fuse.js').FuseResult[]} Results as items with their score */ -export function fuzzySearchGroups(searchValue) { +export function fuzzySearchGroups(searchValue, fuzzySearchCaches) { const keys = [ { name: 'name', weight: 20 }, { name: 'members', weight: 15 }, @@ -1976,7 +1966,7 @@ export function fuzzySearchGroups(searchValue) { { name: 'id', weight: 1 }, ]; - return performFuzzySearch(fuzzySearchCategories.groups, groups, keys, searchValue); + return performFuzzySearch(fuzzySearchCategories.groups, groups, keys, searchValue, fuzzySearchCaches); } /**