rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Encapsulate logic into filters instead of spreading around

Browse Source
This commit is contained in:
Joe
2024-11-26 19:47:09 -08:00
parent 78c55558af
commit 1395c0b8c6
4 changed files with 84 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
docs/fuzzy_search_flow.md
View File

@ -1,113 +0,0 @@
## Input Sources to printCharacters
`printCharacters` is the main function that triggers the fuzzy search process if fuzzy search is enabled.
```mermaid
%%{init: {'flowchart': {'nodeSpacing': 50, 'rankSpacing': 50}}}%%
flowchart TD
subgraph User Actions
UA1[Character Search Input]
UA2[Tag Filter Click]
UA3[Folder Navigation]
UA4[Character Delete]
UA5[Character Create]
UA6[Character Import]
UA7[Clear All Filters]
UA8[Bulk Edit Operations]
UA9[Persona Changes]
end
subgraph API Events
API1[Character List Update]
API2[Group Update]
API3[Tag Update]
end
subgraph System Events
SE1[Page Load]
SE2[Content Manager Update]
SE3[Extension Events]
end
UA1 -->|triggers| PCD[printCharactersDebounced]
UA2 -->|triggers| PCD
UA7 -->|triggers| PCD
UA8 -->|triggers| PCD
UA9 -->|triggers| PCD
UA3 -->|triggers| PC[printCharacters]
UA4 -->|triggers| PC
UA5 -->|triggers| PC
UA6 -->|triggers| PC
API1 -->|triggers| PC
API2 -->|triggers| PC
API3 -->|triggers| PC
SE1 -->|triggers| PC
SE2 -->|triggers| PC
SE3 -->|triggers| PC
PCD -->|debounced call| PC
style PC fill:#f96,stroke:#333
style PCD fill:#f96,stroke:#333
```
This diagram shows how `printCharacters` is called throughout the application:
1. User Actions that trigger character list updates:
- Search input (debounced)
- Tag filter clicks (debounced)
- Folder navigation (direct)
- Character management operations (direct)
2. API Events that require list refresh:
- Character list updates
- Group updates
- Tag system updates
3. System Events:
- Initial page load
- Content manager updates
- Extension-triggered refreshes
## Fuzzy Search Flow
This diagram shows the flow of fuzzy search operations:
```mermaid
sequenceDiagram
participant Data as Data Sources
participant PC as printCharacters
participant GEL as getEntitiesList
participant FH as FilterHelper
participant AF as applyFilters
participant FS as FuzzySearch Functions
participant Cache as FuzzySearchCaches
Note over Data: Changes from:<br/>- Tags<br/>- Personas<br/>- World Info<br/>- Groups
Data->>PC: All changes trigger printCharacters<br/>(direct or debounced)
PC->>GEL: Call with {doFilter: true}
GEL->>FH: filterByTagState(entities)
GEL->>AF: entitiesFilter.applyFilters(entities)
AF->>FH: Check scoreCache for existing results
FH-->>AF: Return cached scores if exist
Note over FS: Filter functions include:<br/>SEARCH, <br/>FAV, <br/>GROUP, <br/>FOLDER, <br/>TAG, <br/>WORLD_INFO_SEARCH, <br/>PERSONA_SEARCH
AF->>FS: fuzzySearchCharacters/Groups/Tags
FS->>Cache: Check/Store results
FS-->>AF: Return search results
AF->>FH: Cache new scores
AF-->>GEL: Return filtered entities
GEL-->>PC: Return final entities list
PC->>Cache: clearFuzzySearchCaches()
Note over Cache: Cache is cleared at the end of<br/>each printCharacters call,<br/>ensuring fresh results for next search
```

10
public/script.js
View File

@ -268,7 +268,6 @@ import { initServerHistory } from './scripts/server-history.js';
import { initSettingsSearch } from './scripts/setting-search.js';
import { initBulkEdit } from './scripts/bulk-edit.js';
import { deriveTemplatesFromChatTemplate } from './scripts/chat-templates.js';
import { clearFuzzySearchCaches } from './scripts/power-user.js';
//exporting functions and vars for mods
export {
@ -1516,7 +1515,6 @@ export async function printCharacters(fullRefresh = false) {
});
favsToHotswap();
clearFuzzySearchCaches();
}
/** Checks the state of the current search, and adds/removes the search sorting option accordingly */
@ -1623,7 +1621,7 @@ export function getEntitiesList({ doFilter = false, doSort = true } = {}) {
subEntities = filterByTagState(entities, { subForEntity: entity });
if (doFilter) {
// sub entities filter "hacked" because folder filter should not be applied there, so even in "only folders" mode characters show up
subEntities = entitiesFilter.applyFilters(subEntities, { clearScoreCache: false, tempOverrides: { [FILTER_TYPES.FOLDER]: FILTER_STATES.UNDEFINED } });
subEntities = entitiesFilter.applyFilters(subEntities, { clearScoreCache: false, tempOverrides: { [FILTER_TYPES.FOLDER]: FILTER_STATES.UNDEFINED }, clearFuzzySearchCaches: false });
}
if (doSort) {
sortEntitiesList(subEntities);
@ -1636,11 +1634,11 @@ export function getEntitiesList({ doFilter = false, doSort = true } = {}) {
// Second run filters, hiding whatever should be filtered later
if (doFilter) {
const beforeFinalEntities = filterByTagState(entities, { globalDisplayFilters: true });
entities = entitiesFilter.applyFilters(beforeFinalEntities);
entities = entitiesFilter.applyFilters(beforeFinalEntities, { clearFuzzySearchCaches: false });
// Magic for folder filter. If that one is enabled, and no folders are display anymore, we remove that filter to actually show the characters.
if (isFilterState(entitiesFilter.getFilterData(FILTER_TYPES.FOLDER), FILTER_STATES.SELECTED) && entities.filter(x => x.type == 'tag').length == 0) {
entities = entitiesFilter.applyFilters(beforeFinalEntities, { tempOverrides: { [FILTER_TYPES.FOLDER]: FILTER_STATES.UNDEFINED } });
entities = entitiesFilter.applyFilters(beforeFinalEntities, { tempOverrides: { [FILTER_TYPES.FOLDER]: FILTER_STATES.UNDEFINED }, clearFuzzySearchCaches: false });
}
}
@ -1656,6 +1654,7 @@ export function getEntitiesList({ doFilter = false, doSort = true } = {}) {
if (doSort) {
sortEntitiesList(entities);
}
entitiesFilter.clearFuzzySearchCaches();
return entities;
}
@ -1751,6 +1750,7 @@ export async function getCharacters() {
}
await getGroups();
// clearFuzzySearchCaches();
await printCharacters(true);
}
}

57
public/scripts/filters.js
View File

@ -55,6 +55,19 @@ export function isFilterState(a, b) {
return aKey === bKey;
}
/**
* The fuzzy search categories
* @type {{ characters: string, worldInfo: string, personas: string, tags: string, groups: string }}
*/
export const fuzzySearchCategories = Object.freeze({
characters: 'characters',
worldInfo: 'worldInfo',
personas: 'personas',
tags: 'tags',
groups: 'groups',
});
/**
* Helper class for filtering data.
* @example
@ -72,6 +85,12 @@ export class FilterHelper {
*/
scoreCache;
/**
* Cache for fuzzy search results per category.
* @type {Object.<string, { resultMap: Map<string, any> }>}
*/
fuzzySearchCaches;
/**
* Creates a new FilterHelper
* @param {Function} onDataChanged Callback to trigger when the filter data changes
@ -79,6 +98,13 @@ export class FilterHelper {
constructor(onDataChanged) {
this.onDataChanged = onDataChanged;
this.scoreCache = new Map();
this.fuzzySearchCaches = {
[fuzzySearchCategories.characters]: { resultMap: new Map() },
[fuzzySearchCategories.worldInfo]: { resultMap: new Map() },
[fuzzySearchCategories.personas]: { resultMap: new Map() },
[fuzzySearchCategories.tags]: { resultMap: new Map() },
[fuzzySearchCategories.groups]: { resultMap: new Map() },
};
}
/**
@ -151,7 +177,7 @@ export class FilterHelper {
return data;
}
const fuzzySearchResults = fuzzySearchWorldInfo(data, term);
const fuzzySearchResults = fuzzySearchWorldInfo(data, term, this.fuzzySearchCaches);
this.cacheScores(FILTER_TYPES.WORLD_INFO_SEARCH, new Map(fuzzySearchResults.map(i => [i.item?.uid, i.score])));
const filteredData = data.filter(entity => fuzzySearchResults.find(x => x.item === entity));
@ -170,7 +196,7 @@ export class FilterHelper {
return data;
}
const fuzzySearchResults = fuzzySearchPersonas(data, term);
const fuzzySearchResults = fuzzySearchPersonas(data, term, this.fuzzySearchCaches);
this.cacheScores(FILTER_TYPES.PERSONA_SEARCH, new Map(fuzzySearchResults.map(i => [i.item.key, i.score])));
const filteredData = data.filter(name => fuzzySearchResults.find(x => x.item.key === name));
@ -289,9 +315,9 @@ export class FilterHelper {
// Save fuzzy search results and scores if enabled
if (power_user.fuzzy_search) {
const fuzzySearchCharactersResults = fuzzySearchCharacters(searchValue);
const fuzzySearchGroupsResults = fuzzySearchGroups(searchValue);
const fuzzySearchTagsResult = fuzzySearchTags(searchValue);
const fuzzySearchCharactersResults = fuzzySearchCharacters(searchValue, this.fuzzySearchCaches);
const fuzzySearchGroupsResults = fuzzySearchGroups(searchValue, this.fuzzySearchCaches);
const fuzzySearchTagsResult = fuzzySearchTags(searchValue, this.fuzzySearchCaches);
this.cacheScores(FILTER_TYPES.SEARCH, new Map(fuzzySearchCharactersResults.map(i => [`character.${i.refIndex}`, i.score])));
this.cacheScores(FILTER_TYPES.SEARCH, new Map(fuzzySearchGroupsResults.map(i => [`group.${i.item.id}`, i.score])));
this.cacheScores(FILTER_TYPES.SEARCH, new Map(fuzzySearchTagsResult.map(i => [`tag.${i.item.id}`, i.score])));
@ -343,11 +369,14 @@ export class FilterHelper {
* @param {object} options - Optional call parameters
* @param {boolean} [options.clearScoreCache=true] - Whether the score cache should be cleared.
* @param {Object.<FilterType, any>} [options.tempOverrides={}] - Temporarily override specific filters for this filter application
* @param {boolean} [options.clearFuzzySearchCaches=true] - Whether the fuzzy search caches should be cleared.
* @returns {any[]} The filtered data.
*/
applyFilters(data, { clearScoreCache = true, tempOverrides = {} } = {}) {
applyFilters(data, { clearScoreCache = true, tempOverrides = {}, clearFuzzySearchCaches = true } = {}) {
if (clearScoreCache) this.clearScoreCache();
if (clearFuzzySearchCaches) this.clearFuzzySearchCaches();
// Save original filter states
const originalStates = {};
for (const key in tempOverrides) {
@ -411,4 +440,20 @@ export class FilterHelper {
this.scoreCache = new Map();
}
}
/**
* Clears fuzzy search caches
* @param {keyof typeof fuzzySearchCategories} [type] Optional cache type to clear. If not provided, clears all caches
*/
clearFuzzySearchCaches(type = null) {
if (type && this.fuzzySearchCaches[type]) {
this.fuzzySearchCaches[type].resultMap.clear();
console.log(`Fuzzy search cache cleared for: ${type}`);
} else {
for (const cache of Object.values(this.fuzzySearchCaches)) {
cache.resultMap.clear();
}
console.log('All fuzzy search caches cleared');
}
}
}

66
public/scripts/power-user.js
View File

@ -53,6 +53,7 @@ import { SlashCommandEnumValue, enumTypes } from './slash-commands/SlashCommandE
import { commonEnumProviders, enumIcons } from './slash-commands/SlashCommandCommonEnumsProvider.js';
import { POPUP_TYPE, callGenericPopup } from './popup.js';
import { loadSystemPrompts } from './sysprompt.js';
import { fuzzySearchCategories } from './filters.js';
export {
loadPowerUserSettings,
@ -328,22 +329,6 @@ const contextControls = [
let browser_has_focus = true;
const debug_functions = [];
const fuzzySearchCaches = {
characters: { resultMap: new Map() },
worldInfo: { resultMap: new Map() },
personas: { resultMap: new Map() },
tags: { resultMap: new Map() },
groups: { resultMap: new Map() },
};
const fuzzySearchCategories = {
characters: 'characters',
worldInfo: 'worldInfo',
personas: 'personas',
tags: 'tags',
groups: 'groups',
};
const setHotswapsDebounced = debounce(favsToHotswap);
function playMessageSound() {
@ -1845,19 +1830,26 @@ async function loadContextSettings() {
});
}
/**
* Common function to perform fuzzy search with caching
* @param {string} type - Type of search from fuzzySearchCategories
* @param {any[]} data - Data array to search in
* @param {Array<{name: string, weight: number, getFn?: Function}>} keys - Fuse.js keys configuration
* @param {string} searchValue - The search term
* @param {Object.<string, { resultMap: Map<string, any> }>} fuzzySearchCaches - Fuzzy search caches
* @returns {import('fuse.js').FuseResult<any>[]} Results as items with their score
*/
function performFuzzySearch(type, data, keys, searchValue) {
function performFuzzySearch(type, data, keys, searchValue, fuzzySearchCaches) {
let startTime = performance.now();
const cache = fuzzySearchCaches[type];
// Check cache for existing results
if (cache.resultMap.has(searchValue)) {
let endTime = performance.now();
if (endTime - startTime > 1.0) {
console.log(`Fuzzy search for ${type} took ${endTime - startTime}ms (cached)`);
}
return cache.resultMap.get(searchValue);
}
@ -1872,26 +1864,20 @@ function performFuzzySearch(type, data, keys, searchValue) {
const results = fuse.search(searchValue);
cache.resultMap.set(searchValue, results);
return results;
}
/**
* Clears all fuzzy search caches
*/
export function clearFuzzySearchCaches() {
for (const cache of Object.values(fuzzySearchCaches)) {
cache.resultMap.clear();
let endTime = performance.now();
if (endTime - startTime > 1.0) {
console.log(`Fuzzy search for ${type} took ${endTime - startTime}ms`);
}
console.log('Fuzzy search caches cleared');
return results;
}
/**
* Fuzzy search characters by a search term
* @param {string} searchValue - The search term
* @param {Object.<string, { resultMap: Map<string, any> }>} fuzzySearchCaches - Fuzzy search caches
* @returns {import('fuse.js').FuseResult<any>[]} Results as items with their score
*/
export function fuzzySearchCharacters(searchValue) {
export function fuzzySearchCharacters(searchValue, fuzzySearchCaches) {
const keys = [
{ name: 'data.name', weight: 20 },
{ name: '#tags', weight: 10, getFn: (character) => getTagsList(character.avatar).map(x => x.name).join('||') },
@ -1906,16 +1892,17 @@ export function fuzzySearchCharacters(searchValue) {
{ name: 'data.alternate_greetings', weight: 1 },
];
return performFuzzySearch(fuzzySearchCategories.characters, characters, keys, searchValue);
return performFuzzySearch(fuzzySearchCategories.characters, characters, keys, searchValue, fuzzySearchCaches);
}
/**
* Fuzzy search world info entries by a search term
* @param {*[]} data - WI items data array
* @param {string} searchValue - The search term
* @param {Object.<string, { resultMap: Map<string, any> }>} fuzzySearchCaches - Fuzzy search caches
* @returns {import('fuse.js').FuseResult<any>[]} Results as items with their score
*/
export function fuzzySearchWorldInfo(data, searchValue) {
export function fuzzySearchWorldInfo(data, searchValue, fuzzySearchCaches) {
const keys = [
{ name: 'key', weight: 20 },
{ name: 'group', weight: 15 },
@ -1926,16 +1913,17 @@ export function fuzzySearchWorldInfo(data, searchValue) {
{ name: 'automationId', weight: 1 },
];
return performFuzzySearch(fuzzySearchCategories.worldInfo, data, keys, searchValue);
return performFuzzySearch(fuzzySearchCategories.worldInfo, data, keys, searchValue, fuzzySearchCaches);
}
/**
* Fuzzy search persona entries by a search term
* @param {*[]} data - persona data array
* @param {string} searchValue - The search term
* @param {Object.<string, { resultMap: Map<string, any> }>} fuzzySearchCaches - Fuzzy search caches
* @returns {import('fuse.js').FuseResult<any>[]} Results as items with their score
*/
export function fuzzySearchPersonas(data, searchValue) {
export function fuzzySearchPersonas(data, searchValue, fuzzySearchCaches) {
const mappedData = data.map(x => ({
key: x,
name: power_user.personas[x] ?? '',
@ -1947,28 +1935,30 @@ export function fuzzySearchPersonas(data, searchValue) {
{ name: 'description', weight: 3 },
];
return performFuzzySearch(fuzzySearchCategories.personas, mappedData, keys, searchValue);
return performFuzzySearch(fuzzySearchCategories.personas, mappedData, keys, searchValue, fuzzySearchCaches);
}
/**
* Fuzzy search tags by a search term
* @param {string} searchValue - The search term
* @param {Object.<string, { resultMap: Map<string, any> }>} fuzzySearchCaches - Fuzzy search caches
* @returns {import('fuse.js').FuseResult<any>[]} Results as items with their score
*/
export function fuzzySearchTags(searchValue) {
export function fuzzySearchTags(searchValue, fuzzySearchCaches) {
const keys = [
{ name: 'name', weight: 1 },
];
return performFuzzySearch(fuzzySearchCategories.tags, tags, keys, searchValue);
return performFuzzySearch(fuzzySearchCategories.tags, tags, keys, searchValue, fuzzySearchCaches);
}
/**
* Fuzzy search groups by a search term
* @param {string} searchValue - The search term
* @param {Object.<string, { resultMap: Map<string, any> }>} fuzzySearchCaches - Fuzzy search caches
* @returns {import('fuse.js').FuseResult<any>[]} Results as items with their score
*/
export function fuzzySearchGroups(searchValue) {
export function fuzzySearchGroups(searchValue, fuzzySearchCaches) {
const keys = [
{ name: 'name', weight: 20 },
{ name: 'members', weight: 15 },
@ -1976,7 +1966,7 @@ export function fuzzySearchGroups(searchValue) {
{ name: 'id', weight: 1 },
];
return performFuzzySearch(fuzzySearchCategories.groups, groups, keys, searchValue);
return performFuzzySearch(fuzzySearchCategories.groups, groups, keys, searchValue, fuzzySearchCaches);
}
/**