diff --git a/default/config.yaml b/default/config.yaml index 16248ef83..f4bacf9ff 100644 --- a/default/config.yaml +++ b/default/config.yaml @@ -1,8 +1,6 @@ # -- DATA CONFIGURATION -- # Root directory for user data storage dataRoot: ./data -# The maximum amount of memory that parsed character cards can use in MB -cardsCacheCapacity: 100 # -- SERVER CONFIGURATION -- # Listen for incoming connections listen: false @@ -135,6 +133,14 @@ thumbnails: # Maximum thumbnail dimensions per type [width, height] dimensions: { 'bg': [160, 90], 'avatar': [96, 144] } +# PERFORMANCE-RELATED CONFIGURATION +performance: + # Enables lazy loading of character cards. Improves performances with large card libraries. + # May have compatibility issues with some extensions. + lazyLoadCharacters: false + # The maximum amount of memory that parsed character cards can use. Set to 0 to disable memory caching. + memoryCacheCapacity: '100mb' + # Allow secret keys exposure via API allowKeysExposure: false # Skip new default content checks diff --git a/package-lock.json b/package-lock.json index a626f6681..0e97d11c7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "bing-translate-api": "^4.0.2", "body-parser": "^1.20.2", "bowser": "^2.11.0", + "bytes": "^3.1.2", "chalk": "^5.4.1", "command-exists": "^1.2.9", "compression": "^1.8.0", @@ -83,6 +84,7 @@ }, "devDependencies": { "@types/archiver": "^6.0.3", + "@types/bytes": "^3.1.5", "@types/command-exists": "^1.2.3", "@types/compression": "^1.7.5", "@types/cookie-parser": "^1.4.8", @@ -1105,6 +1107,13 @@ "@types/node": "*" } }, + "node_modules/@types/bytes": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/@types/bytes/-/bytes-3.1.5.tgz", + "integrity": "sha512-VgZkrJckypj85YxEsEavcMmmSOIzkUHqWmM4CCyia5dc54YwsXzJ5uT4fYxBQNEXx+oF1krlhgCbvfubXqZYsQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/cacheable-request": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz", diff --git a/package.json b/package.json index 9cab7d59b..2a9009aae 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "bing-translate-api": "^4.0.2", "body-parser": "^1.20.2", "bowser": "^2.11.0", + "bytes": "^3.1.2", "chalk": "^5.4.1", "command-exists": "^1.2.9", "compression": "^1.8.0", @@ -92,8 +93,8 @@ "version": "1.12.12", "scripts": { "start": "node server.js", + "debug": "node --inspect server.js", "electron": "electron ./src/electron", - "debug": "node server.js --inspect", "start:deno": "deno run --allow-run --allow-net --allow-read --allow-write --allow-sys --allow-env server.js", "start:bun": "bun server.js", "start:no-csrf": "node server.js --disableCsrf", @@ -113,6 +114,7 @@ "main": "server.js", "devDependencies": { "@types/archiver": "^6.0.3", + "@types/bytes": "^3.1.5", "@types/command-exists": "^1.2.3", "@types/compression": "^1.7.5", "@types/cookie-parser": "^1.4.8", diff --git a/post-install.js b/post-install.js index 1989b61aa..0048d9c6f 100644 --- a/post-install.js +++ b/post-install.js @@ -96,6 +96,11 @@ const keyMigrationMap = [ newKey: 'logging.minLogLevel', migrate: (value) => value, }, + { + oldKey: 'cardsCacheCapacity', + newKey: 'performance.memoryCacheCapacity', + migrate: (value) => `${value}mb`, + }, // uncomment one release after 1.12.13 /* { diff --git a/public/script.js b/public/script.js index 950572210..c69b734e1 100644 --- a/public/script.js +++ b/public/script.js @@ -1780,9 +1780,7 @@ export async function getCharacters() { const response = await fetch('/api/characters/all', { method: 'POST', headers: getRequestHeaders(), - body: JSON.stringify({ - '': '', - }), + body: JSON.stringify({}), }); if (response.ok === true) { characters.splice(0, characters.length); @@ -3678,6 +3676,9 @@ export async function Generate(type, { automatic_trigger, force_name2, quiet_pro setGenerationProgress(0); generation_started = new Date(); + // Prevent generation from shallow characters + await unshallowCharacter(this_chid); + // Occurs every time, even if the generation is aborted due to slash commands execution await eventSource.emit(event_types.GENERATION_STARTED, type, { automatic_trigger, force_name2, quiet_prompt, quietToLoud, skipWIAN, force_chid, signal, quietImage }, dryRun); @@ -6724,9 +6725,43 @@ export function buildAvatarList(block, entities, { templateId = 'inline_avatar_t } } +/** + * Loads all the data of a shallow character. + * @param {string|undefined} characterId Array index + * @returns {Promise} Promise that resolves when the character is unshallowed + */ +export async function unshallowCharacter(characterId) { + if (characterId === undefined) { + console.warn('Undefined character cannot be unshallowed'); + return; + } + + /** @type {import('./scripts/char-data.js').v1CharData} */ + const character = characters[characterId]; + if (!character) { + console.warn('Character not found:', characterId); + return; + } + + // Character is not shallow + if (!character.shallow) { + return; + } + + const avatar = character.avatar; + if (!avatar) { + console.warn('Character has no avatar field:', characterId); + return; + } + + await getOneCharacter(avatar); +} + export async function getChat() { //console.log('/api/chats/get -- entered for -- ' + characters[this_chid].name); try { + await unshallowCharacter(this_chid); + const response = await $.ajax({ type: 'POST', url: '/api/chats/get', diff --git a/public/scripts/RossAscends-mods.js b/public/scripts/RossAscends-mods.js index d7fcb1c5a..0a2581e5f 100644 --- a/public/scripts/RossAscends-mods.js +++ b/public/scripts/RossAscends-mods.js @@ -316,7 +316,10 @@ export async function favsToHotswap() { const entities = getEntitiesList({ doFilter: false }); const container = $('#right-nav-panel .hotswap'); - const favs = entities.filter(x => x.item.fav || x.item.fav == 'true'); + // Hard limit is required because even if all hotswaps don't fit the screen, their images would still be loaded + // 25 is roughly calculated as the maximum number of favs that can fit an ultrawide monitor with the default theme + const FAVS_LIMIT = 25; + const favs = entities.filter(x => x.item.fav || x.item.fav == 'true').slice(0, FAVS_LIMIT); //helpful instruction message if no characters are favorited if (favs.length == 0) { diff --git a/public/scripts/char-data.js b/public/scripts/char-data.js index be28d8172..51e85722c 100644 --- a/public/scripts/char-data.js +++ b/public/scripts/char-data.js @@ -113,5 +113,6 @@ * @property {string} chat - name of the current chat file chat * @property {string} avatar - file name of the avatar image (acts as a unique identifier) * @property {string} json_data - the full raw JSON data of the character + * @property {boolean?} shallow - if the data is shallow (lazy-loaded) */ export default 0;// now this file is a module diff --git a/public/scripts/group-chats.js b/public/scripts/group-chats.js index d1431d74f..0fbd5b543 100644 --- a/public/scripts/group-chats.js +++ b/public/scripts/group-chats.js @@ -72,6 +72,7 @@ import { animation_duration, depth_prompt_role_default, shouldAutoContinue, + unshallowCharacter, } from '../script.js'; import { printTagList, createTagMapFromList, applyTagsOnCharacterSelect, tag_map, applyTagsOnGroupSelect } from './tags.js'; import { FILTER_TYPES, FilterHelper } from './filters.js'; @@ -216,6 +217,7 @@ export async function getGroupChat(groupId, reload = false) { // Run validation before any loading validateGroup(group); + await unshallowGroupMembers(groupId); const chat_id = group.chat_id; const data = await loadGroupChat(chat_id); @@ -824,6 +826,8 @@ async function generateGroupWrapper(by_auto_mode, type = null, params = {}) { } try { + await unshallowGroupMembers(selected_group); + throwIfAborted(); hideSwipeButtons(); is_group_generating = true; @@ -1137,6 +1141,29 @@ export async function editGroup(id, immediately, reload = true) { saveGroupDebounced(group, reload); } +/** + * Unshallows all definitions of group members. + * @param {string} groupId Id of the group + * @returns {Promise} Promise that resolves when all group members are unshallowed + */ +export async function unshallowGroupMembers(groupId) { + const group = groups.find(x => x.id == groupId); + if (!group) { + return; + } + const members = group.members; + if (!Array.isArray(members)) { + return; + } + for (const member of members) { + const index = characters.findIndex(x => x.avatar === member); + if (index === -1) { + continue; + } + await unshallowCharacter(String(index)); + } +} + let groupAutoModeAbortController = null; async function groupChatAutoModeWorker() { @@ -1158,9 +1185,9 @@ async function groupChatAutoModeWorker() { await generateGroupWrapper(true, 'auto', { signal: groupAutoModeAbortController.signal }); } -async function modifyGroupMember(chat_id, groupMember, isDelete) { +async function modifyGroupMember(groupId, groupMember, isDelete) { const id = groupMember.data('id'); - const thisGroup = groups.find((x) => x.id == chat_id); + const thisGroup = groups.find((x) => x.id == groupId); const membersArray = thisGroup?.members ?? newGroupMembers; if (isDelete) { @@ -1173,6 +1200,7 @@ async function modifyGroupMember(chat_id, groupMember, isDelete) { } if (openGroupId) { + await unshallowGroupMembers(openGroupId); await editGroup(openGroupId, false, false); updateGroupAvatar(thisGroup); } @@ -1638,7 +1666,7 @@ async function onGroupActionClick(event) { } if (action === 'view') { - openCharacterDefinition(member); + await openCharacterDefinition(member); } if (action === 'speak') { @@ -1690,7 +1718,7 @@ export async function openGroupById(groupId) { return false; } -function openCharacterDefinition(characterSelect) { +async function openCharacterDefinition(characterSelect) { if (is_group_generating) { toastr.warning(t`Can't peek a character while group reply is being generated`); console.warn('Can\'t peek a character def while group reply is being generated'); @@ -1703,6 +1731,7 @@ function openCharacterDefinition(characterSelect) { return; } + await unshallowCharacter(chid); setCharacterId(chid); select_selected_character(chid); // Gentle nudge to recalculate tokens diff --git a/public/scripts/st-context.js b/public/scripts/st-context.js index 5278e7591..cfe55d379 100644 --- a/public/scripts/st-context.js +++ b/public/scripts/st-context.js @@ -47,6 +47,7 @@ import { updateMessageBlock, printMessages, clearChat, + unshallowCharacter, } from '../script.js'; import { extension_settings, @@ -55,7 +56,7 @@ import { renderExtensionTemplateAsync, writeExtensionField, } from './extensions.js'; -import { groups, openGroupChat, selected_group } from './group-chats.js'; +import { groups, openGroupChat, selected_group, unshallowGroupMembers } from './group-chats.js'; import { addLocaleData, getCurrentLocale, t, translate } from './i18n.js'; import { hideLoader, showLoader } from './loader.js'; import { MacrosParser } from './macros.js'; @@ -210,6 +211,8 @@ export function getContext() { clearChat, ChatCompletionService, TextCompletionService, + unshallowCharacter, + unshallowGroupMembers, }; } diff --git a/src/endpoints/characters.js b/src/endpoints/characters.js index f4a7307a5..a4ec7b34d 100644 --- a/src/endpoints/characters.js +++ b/src/endpoints/characters.js @@ -23,12 +23,13 @@ import { invalidateThumbnail } from './thumbnails.js'; import { importRisuSprites } from './sprites.js'; const defaultAvatarPath = './public/img/ai4.png'; -// KV-store for parsed character data -const cacheCapacity = Number(getConfigValue('cardsCacheCapacity', 100, 'number')); // MB // With 100 MB limit it would take roughly 3000 characters to reach this limit -const characterDataCache = new MemoryLimitedMap(1024 * 1024 * cacheCapacity); +const memoryCacheCapacity = getConfigValue('performance.memoryCacheCapacity', '100mb'); +const memoryCache = new MemoryLimitedMap(memoryCacheCapacity); // Some Android devices require tighter memory management const isAndroid = process.platform === 'android'; +// Use shallow character data for the character list +const useShallowCharacters = !!getConfigValue('performance.lazyLoadCharacters', false, 'boolean'); /** * Reads the character card from the specified image file. @@ -39,12 +40,12 @@ const isAndroid = process.platform === 'android'; async function readCharacterData(inputFile, inputFormat = 'png') { const stat = fs.statSync(inputFile); const cacheKey = `${inputFile}-${stat.mtimeMs}`; - if (characterDataCache.has(cacheKey)) { - return characterDataCache.get(cacheKey); + if (memoryCache.has(cacheKey)) { + return memoryCache.get(cacheKey); } const result = parse(inputFile, inputFormat); - !isAndroid && characterDataCache.set(cacheKey, result); + !isAndroid && memoryCache.set(cacheKey, result); return result; } @@ -60,12 +61,12 @@ async function readCharacterData(inputFile, inputFormat = 'png') { async function writeCharacterData(inputFile, data, outputFile, request, crop = undefined) { try { // Reset the cache - for (const key of characterDataCache.keys()) { + for (const key of memoryCache.keys()) { if (Buffer.isBuffer(inputFile)) { break; } if (key.startsWith(inputFile)) { - characterDataCache.delete(key); + memoryCache.delete(key); break; } } @@ -200,14 +201,45 @@ const calculateDataSize = (data) => { return typeof data === 'object' ? Object.values(data).reduce((acc, val) => acc + String(val).length, 0) : 0; }; +/** + * Only get fields that are used to display the character list. + * @param {object} character Character object + * @returns {{shallow: true, [key: string]: any}} Shallow character + */ +const toShallow = (character) => { + return { + shallow: true, + name: character.name, + avatar: character.avatar, + chat: character.chat, + fav: character.fav, + date_added: character.date_added, + create_date: character.create_date, + date_last_chat: character.date_last_chat, + chat_size: character.chat_size, + data_size: character.data_size, + data: { + name: _.get(character, 'data.name', ''), + character_version: _.get(character, 'data.character_version', ''), + creator: _.get(character, 'data.creator', ''), + creator_notes: _.get(character, 'data.creator_notes', ''), + extensions: { + fav: _.get(character, 'data.extensions.fav', false), + }, + }, + }; +}; + /** * processCharacter - Process a given character, read its data and calculate its statistics. * * @param {string} item The name of the character. * @param {import('../users.js').UserDirectoryList} directories User directories + * @param {object} options Options for the character processing + * @param {boolean} options.shallow If true, only return the core character's metadata * @return {Promise} A Promise that resolves when the character processing is done. */ -const processCharacter = async (item, directories) => { +const processCharacter = async (item, directories, { shallow }) => { try { const imgFile = path.join(directories.characters, item); const imgData = await readCharacterData(imgFile); @@ -226,7 +258,7 @@ const processCharacter = async (item, directories) => { character['chat_size'] = chatSize; character['date_last_chat'] = dateLastChat; character['data_size'] = calculateDataSize(jsonObject?.data); - return character; + return shallow ? toShallow(character) : character; } catch (err) { console.error(`Could not process character: ${item}`); @@ -993,7 +1025,7 @@ router.post('/all', jsonParser, async function (request, response) { try { const files = fs.readdirSync(request.user.directories.characters); const pngFiles = files.filter(file => file.endsWith('.png')); - const processingPromises = pngFiles.map(file => processCharacter(file, request.user.directories)); + const processingPromises = pngFiles.map(file => processCharacter(file, request.user.directories, { shallow: useShallowCharacters })); const data = (await Promise.all(processingPromises)).filter(c => c.name); return response.send(data); } catch (err) { @@ -1012,7 +1044,7 @@ router.post('/get', jsonParser, validateAvatarUrlMiddleware, async function (req return response.sendStatus(404); } - const data = await processCharacter(item, request.user.directories); + const data = await processCharacter(item, request.user.directories, { shallow: false }); return response.send(data); } catch (err) { @@ -1022,11 +1054,11 @@ router.post('/get', jsonParser, validateAvatarUrlMiddleware, async function (req }); router.post('/chats', jsonParser, validateAvatarUrlMiddleware, async function (request, response) { - if (!request.body) return response.sendStatus(400); - - const characterDirectory = (request.body.avatar_url).replace('.png', ''); - try { + if (!request.body) return response.sendStatus(400); + + const characterDirectory = (request.body.avatar_url).replace('.png', ''); + const chatsDirectory = path.join(request.user.directories.chats, characterDirectory); if (!fs.existsSync(chatsDirectory)) { diff --git a/src/util.js b/src/util.js index e6246b5d2..1dec2b466 100644 --- a/src/util.js +++ b/src/util.js @@ -16,6 +16,7 @@ import mime from 'mime-types'; import { default as simpleGit } from 'simple-git'; import chalk from 'chalk'; import { LOG_LEVELS } from './constants.js'; +import bytes from 'bytes'; /** * Parsed config object. @@ -856,14 +857,10 @@ export function setupLogLevel() { export class MemoryLimitedMap { /** * Creates an instance of MemoryLimitedMap. - * @param {number} maxMemoryInBytes - The maximum allowed memory in bytes for string values. + * @param {string} cacheCapacity - Maximum memory usage in human-readable format (e.g., '1 GB'). */ - constructor(maxMemoryInBytes) { - if (typeof maxMemoryInBytes !== 'number' || maxMemoryInBytes <= 0 || isNaN(maxMemoryInBytes)) { - console.warn('Invalid maxMemoryInBytes, using a fallback value of 1 GB.'); - maxMemoryInBytes = 1024 * 1024 * 1024; // 1 GB - } - this.maxMemory = maxMemoryInBytes; + constructor(cacheCapacity) { + this.maxMemory = bytes.parse(cacheCapacity) ?? 0; this.currentMemory = 0; this.map = new Map(); this.queue = []; @@ -886,6 +883,10 @@ export class MemoryLimitedMap { * @param {string} value */ set(key, value) { + if (this.maxMemory <= 0) { + return; + } + if (typeof key !== 'string' || typeof value !== 'string') { return; }