Merge pull request #3614 from SillyTavern/char-shallow

Lazy load characters
This commit is contained in:
Cohee
2025-03-06 20:57:24 +02:00
committed by GitHub
11 changed files with 161 additions and 35 deletions

View File

@@ -1,8 +1,6 @@
# -- DATA CONFIGURATION --
# Root directory for user data storage
dataRoot: ./data
# The maximum amount of memory that parsed character cards can use in MB
cardsCacheCapacity: 100
# -- SERVER CONFIGURATION --
# Listen for incoming connections
listen: false
@@ -135,6 +133,14 @@ thumbnails:
# Maximum thumbnail dimensions per type [width, height]
dimensions: { 'bg': [160, 90], 'avatar': [96, 144] }
# PERFORMANCE-RELATED CONFIGURATION
performance:
# Enables lazy loading of character cards. Improves performances with large card libraries.
# May have compatibility issues with some extensions.
lazyLoadCharacters: false
# The maximum amount of memory that parsed character cards can use. Set to 0 to disable memory caching.
memoryCacheCapacity: '100mb'
# Allow secret keys exposure via API
allowKeysExposure: false
# Skip new default content checks

9
package-lock.json generated
View File

@@ -21,6 +21,7 @@
"bing-translate-api": "^4.0.2",
"body-parser": "^1.20.2",
"bowser": "^2.11.0",
"bytes": "^3.1.2",
"chalk": "^5.4.1",
"command-exists": "^1.2.9",
"compression": "^1.8.0",
@@ -83,6 +84,7 @@
},
"devDependencies": {
"@types/archiver": "^6.0.3",
"@types/bytes": "^3.1.5",
"@types/command-exists": "^1.2.3",
"@types/compression": "^1.7.5",
"@types/cookie-parser": "^1.4.8",
@@ -1105,6 +1107,13 @@
"@types/node": "*"
}
},
"node_modules/@types/bytes": {
"version": "3.1.5",
"resolved": "https://registry.npmjs.org/@types/bytes/-/bytes-3.1.5.tgz",
"integrity": "sha512-VgZkrJckypj85YxEsEavcMmmSOIzkUHqWmM4CCyia5dc54YwsXzJ5uT4fYxBQNEXx+oF1krlhgCbvfubXqZYsQ==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/cacheable-request": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz",

View File

@@ -11,6 +11,7 @@
"bing-translate-api": "^4.0.2",
"body-parser": "^1.20.2",
"bowser": "^2.11.0",
"bytes": "^3.1.2",
"chalk": "^5.4.1",
"command-exists": "^1.2.9",
"compression": "^1.8.0",
@@ -92,8 +93,8 @@
"version": "1.12.12",
"scripts": {
"start": "node server.js",
"debug": "node --inspect server.js",
"electron": "electron ./src/electron",
"debug": "node server.js --inspect",
"start:deno": "deno run --allow-run --allow-net --allow-read --allow-write --allow-sys --allow-env server.js",
"start:bun": "bun server.js",
"start:no-csrf": "node server.js --disableCsrf",
@@ -113,6 +114,7 @@
"main": "server.js",
"devDependencies": {
"@types/archiver": "^6.0.3",
"@types/bytes": "^3.1.5",
"@types/command-exists": "^1.2.3",
"@types/compression": "^1.7.5",
"@types/cookie-parser": "^1.4.8",

View File

@@ -96,6 +96,11 @@ const keyMigrationMap = [
newKey: 'logging.minLogLevel',
migrate: (value) => value,
},
{
oldKey: 'cardsCacheCapacity',
newKey: 'performance.memoryCacheCapacity',
migrate: (value) => `${value}mb`,
},
// uncomment one release after 1.12.13
/*
{

View File

@@ -1780,9 +1780,7 @@ export async function getCharacters() {
const response = await fetch('/api/characters/all', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({
'': '',
}),
body: JSON.stringify({}),
});
if (response.ok === true) {
characters.splice(0, characters.length);
@@ -3678,6 +3676,9 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro
setGenerationProgress(0);
generation_started = new Date();
// Prevent generation from shallow characters
await unshallowCharacter(this_chid);
// Occurs every time, even if the generation is aborted due to slash commands execution
await eventSource.emit(event_types.GENERATION_STARTED, type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage }, dryRun);
@@ -6724,9 +6725,43 @@ export function buildAvatarList(block, entities, { templateId = 'inline_avatar_t
}
}
/**
* Loads all the data of a shallow character.
* @param {string|undefined} characterId Array index
* @returns {Promise<void>} Promise that resolves when the character is unshallowed
*/
export async function unshallowCharacter(characterId) {
if (characterId === undefined) {
console.warn('Undefined character cannot be unshallowed');
return;
}
/** @type {import('./scripts/char-data.js').v1CharData} */
const character = characters[characterId];
if (!character) {
console.warn('Character not found:', characterId);
return;
}
// Character is not shallow
if (!character.shallow) {
return;
}
const avatar = character.avatar;
if (!avatar) {
console.warn('Character has no avatar field:', characterId);
return;
}
await getOneCharacter(avatar);
}
export async function getChat() {
//console.log('/api/chats/get -- entered for -- ' + characters[this_chid].name);
try {
await unshallowCharacter(this_chid);
const response = await $.ajax({
type: 'POST',
url: '/api/chats/get',

View File

@@ -316,7 +316,10 @@ export async function favsToHotswap() {
const entities = getEntitiesList({ doFilter: false });
const container = $('#right-nav-panel .hotswap');
const favs = entities.filter(x => x.item.fav || x.item.fav == 'true');
// Hard limit is required because even if all hotswaps don't fit the screen, their images would still be loaded
// 25 is roughly calculated as the maximum number of favs that can fit an ultrawide monitor with the default theme
const FAVS_LIMIT = 25;
const favs = entities.filter(x => x.item.fav || x.item.fav == 'true').slice(0, FAVS_LIMIT);
//helpful instruction message if no characters are favorited
if (favs.length == 0) {

View File

@@ -113,5 +113,6 @@
* @property {string} chat - name of the current chat file chat
* @property {string} avatar - file name of the avatar image (acts as a unique identifier)
* @property {string} json_data - the full raw JSON data of the character
* @property {boolean?} shallow - if the data is shallow (lazy-loaded)
*/
export default 0;// now this file is a module

View File

@@ -72,6 +72,7 @@ import {
animation_duration,
depth_prompt_role_default,
shouldAutoContinue,
unshallowCharacter,
} from '../script.js';
import { printTagList, createTagMapFromList, applyTagsOnCharacterSelect, tag_map, applyTagsOnGroupSelect } from './tags.js';
import { FILTER_TYPES, FilterHelper } from './filters.js';
@@ -216,6 +217,7 @@ export async function getGroupChat(groupId, reload = false) {
// Run validation before any loading
validateGroup(group);
await unshallowGroupMembers(groupId);
const chat_id = group.chat_id;
const data = await loadGroupChat(chat_id);
@@ -824,6 +826,8 @@ async function generateGroupWrapper(by_auto_mode, type = null, params = {}) {
}
try {
await unshallowGroupMembers(selected_group);
throwIfAborted();
hideSwipeButtons();
is_group_generating = true;
@@ -1137,6 +1141,29 @@ export async function editGroup(id, immediately, reload = true) {
saveGroupDebounced(group, reload);
}
/**
* Unshallows all definitions of group members.
* @param {string} groupId Id of the group
* @returns {Promise<void>} Promise that resolves when all group members are unshallowed
*/
export async function unshallowGroupMembers(groupId) {
const group = groups.find(x => x.id == groupId);
if (!group) {
return;
}
const members = group.members;
if (!Array.isArray(members)) {
return;
}
for (const member of members) {
const index = characters.findIndex(x => x.avatar === member);
if (index === -1) {
continue;
}
await unshallowCharacter(String(index));
}
}
let groupAutoModeAbortController = null;
async function groupChatAutoModeWorker() {
@@ -1158,9 +1185,9 @@ async function groupChatAutoModeWorker() {
await generateGroupWrapper(true, 'auto', { signal: groupAutoModeAbortController.signal });
}
async function modifyGroupMember(chat_id, groupMember, isDelete) {
async function modifyGroupMember(groupId, groupMember, isDelete) {
const id = groupMember.data('id');
const thisGroup = groups.find((x) => x.id == chat_id);
const thisGroup = groups.find((x) => x.id == groupId);
const membersArray = thisGroup?.members ?? newGroupMembers;
if (isDelete) {
@@ -1173,6 +1200,7 @@ async function modifyGroupMember(chat_id, groupMember, isDelete) {
}
if (openGroupId) {
await unshallowGroupMembers(openGroupId);
await editGroup(openGroupId, false, false);
updateGroupAvatar(thisGroup);
}
@@ -1638,7 +1666,7 @@ async function onGroupActionClick(event) {
}
if (action === 'view') {
openCharacterDefinition(member);
await openCharacterDefinition(member);
}
if (action === 'speak') {
@@ -1690,7 +1718,7 @@ export async function openGroupById(groupId) {
return false;
}
function openCharacterDefinition(characterSelect) {
async function openCharacterDefinition(characterSelect) {
if (is_group_generating) {
toastr.warning(t`Can't peek a character while group reply is being generated`);
console.warn('Can\'t peek a character def while group reply is being generated');
@@ -1703,6 +1731,7 @@ function openCharacterDefinition(characterSelect) {
return;
}
await unshallowCharacter(chid);
setCharacterId(chid);
select_selected_character(chid);
// Gentle nudge to recalculate tokens

View File

@@ -47,6 +47,7 @@ import {
updateMessageBlock,
printMessages,
clearChat,
unshallowCharacter,
} from '../script.js';
import {
extension_settings,
@@ -55,7 +56,7 @@ import {
renderExtensionTemplateAsync,
writeExtensionField,
} from './extensions.js';
import { groups, openGroupChat, selected_group } from './group-chats.js';
import { groups, openGroupChat, selected_group, unshallowGroupMembers } from './group-chats.js';
import { addLocaleData, getCurrentLocale, t, translate } from './i18n.js';
import { hideLoader, showLoader } from './loader.js';
import { MacrosParser } from './macros.js';
@@ -210,6 +211,8 @@ export function getContext() {
clearChat,
ChatCompletionService,
TextCompletionService,
unshallowCharacter,
unshallowGroupMembers,
};
}

View File

@@ -23,12 +23,13 @@ import { invalidateThumbnail } from './thumbnails.js';
import { importRisuSprites } from './sprites.js';
const defaultAvatarPath = './public/img/ai4.png';
// KV-store for parsed character data
const cacheCapacity = Number(getConfigValue('cardsCacheCapacity', 100, 'number')); // MB
// With 100 MB limit it would take roughly 3000 characters to reach this limit
const characterDataCache = new MemoryLimitedMap(1024 * 1024 * cacheCapacity);
const memoryCacheCapacity = getConfigValue('performance.memoryCacheCapacity', '100mb');
const memoryCache = new MemoryLimitedMap(memoryCacheCapacity);
// Some Android devices require tighter memory management
const isAndroid = process.platform === 'android';
// Use shallow character data for the character list
const useShallowCharacters = !!getConfigValue('performance.lazyLoadCharacters', false, 'boolean');
/**
* Reads the character card from the specified image file.
@@ -39,12 +40,12 @@ const isAndroid = process.platform === 'android';
async function readCharacterData(inputFile, inputFormat = 'png') {
const stat = fs.statSync(inputFile);
const cacheKey = `${inputFile}-${stat.mtimeMs}`;
if (characterDataCache.has(cacheKey)) {
return characterDataCache.get(cacheKey);
if (memoryCache.has(cacheKey)) {
return memoryCache.get(cacheKey);
}
const result = parse(inputFile, inputFormat);
!isAndroid && characterDataCache.set(cacheKey, result);
!isAndroid && memoryCache.set(cacheKey, result);
return result;
}
@@ -60,12 +61,12 @@ async function readCharacterData(inputFile, inputFormat = 'png') {
async function writeCharacterData(inputFile, data, outputFile, request, crop = undefined) {
try {
// Reset the cache
for (const key of characterDataCache.keys()) {
for (const key of memoryCache.keys()) {
if (Buffer.isBuffer(inputFile)) {
break;
}
if (key.startsWith(inputFile)) {
characterDataCache.delete(key);
memoryCache.delete(key);
break;
}
}
@@ -200,14 +201,45 @@ const calculateDataSize = (data) => {
return typeof data === 'object' ? Object.values(data).reduce((acc, val) => acc + String(val).length, 0) : 0;
};
/**
* Only get fields that are used to display the character list.
* @param {object} character Character object
* @returns {{shallow: true, [key: string]: any}} Shallow character
*/
const toShallow = (character) => {
return {
shallow: true,
name: character.name,
avatar: character.avatar,
chat: character.chat,
fav: character.fav,
date_added: character.date_added,
create_date: character.create_date,
date_last_chat: character.date_last_chat,
chat_size: character.chat_size,
data_size: character.data_size,
data: {
name: _.get(character, 'data.name', ''),
character_version: _.get(character, 'data.character_version', ''),
creator: _.get(character, 'data.creator', ''),
creator_notes: _.get(character, 'data.creator_notes', ''),
extensions: {
fav: _.get(character, 'data.extensions.fav', false),
},
},
};
};
/**
* processCharacter - Process a given character, read its data and calculate its statistics.
*
* @param {string} item The name of the character.
* @param {import('../users.js').UserDirectoryList} directories User directories
* @param {object} options Options for the character processing
* @param {boolean} options.shallow If true, only return the core character's metadata
* @return {Promise<object>} A Promise that resolves when the character processing is done.
*/
const processCharacter = async (item, directories) => {
const processCharacter = async (item, directories, { shallow }) => {
try {
const imgFile = path.join(directories.characters, item);
const imgData = await readCharacterData(imgFile);
@@ -226,7 +258,7 @@ const processCharacter = async (item, directories) => {
character['chat_size'] = chatSize;
character['date_last_chat'] = dateLastChat;
character['data_size'] = calculateDataSize(jsonObject?.data);
return character;
return shallow ? toShallow(character) : character;
}
catch (err) {
console.error(`Could not process character: ${item}`);
@@ -993,7 +1025,7 @@ router.post('/all', jsonParser, async function (request, response) {
try {
const files = fs.readdirSync(request.user.directories.characters);
const pngFiles = files.filter(file => file.endsWith('.png'));
const processingPromises = pngFiles.map(file => processCharacter(file, request.user.directories));
const processingPromises = pngFiles.map(file => processCharacter(file, request.user.directories, { shallow: useShallowCharacters }));
const data = (await Promise.all(processingPromises)).filter(c => c.name);
return response.send(data);
} catch (err) {
@@ -1012,7 +1044,7 @@ router.post('/get', jsonParser, validateAvatarUrlMiddleware, async function (req
return response.sendStatus(404);
}
const data = await processCharacter(item, request.user.directories);
const data = await processCharacter(item, request.user.directories, { shallow: false });
return response.send(data);
} catch (err) {
@@ -1022,11 +1054,11 @@ router.post('/get', jsonParser, validateAvatarUrlMiddleware, async function (req
});
router.post('/chats', jsonParser, validateAvatarUrlMiddleware, async function (request, response) {
if (!request.body) return response.sendStatus(400);
const characterDirectory = (request.body.avatar_url).replace('.png', '');
try {
if (!request.body) return response.sendStatus(400);
const characterDirectory = (request.body.avatar_url).replace('.png', '');
const chatsDirectory = path.join(request.user.directories.chats, characterDirectory);
if (!fs.existsSync(chatsDirectory)) {

View File

@@ -16,6 +16,7 @@ import mime from 'mime-types';
import { default as simpleGit } from 'simple-git';
import chalk from 'chalk';
import { LOG_LEVELS } from './constants.js';
import bytes from 'bytes';
/**
* Parsed config object.
@@ -856,14 +857,10 @@ export function setupLogLevel() {
export class MemoryLimitedMap {
/**
* Creates an instance of MemoryLimitedMap.
* @param {number} maxMemoryInBytes - The maximum allowed memory in bytes for string values.
* @param {string} cacheCapacity - Maximum memory usage in human-readable format (e.g., '1 GB').
*/
constructor(maxMemoryInBytes) {
if (typeof maxMemoryInBytes !== 'number' || maxMemoryInBytes <= 0 || isNaN(maxMemoryInBytes)) {
console.warn('Invalid maxMemoryInBytes, using a fallback value of 1 GB.');
maxMemoryInBytes = 1024 * 1024 * 1024; // 1 GB
}
this.maxMemory = maxMemoryInBytes;
constructor(cacheCapacity) {
this.maxMemory = bytes.parse(cacheCapacity) ?? 0;
this.currentMemory = 0;
this.map = new Map();
this.queue = [];
@@ -886,6 +883,10 @@ export class MemoryLimitedMap {
* @param {string} value
*/
set(key, value) {
if (this.maxMemory <= 0) {
return;
}
if (typeof key !== 'string' || typeof value !== 'string') {
return;
}