From f75daba6c0206f80e1d1a048df9546de5b847b9a Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 21 Apr 2024 23:38:18 +0300 Subject: [PATCH 01/21] Image inlining hint always visible --- public/css/toggle-dependent.css | 8 -------- 1 file changed, 8 deletions(-) diff --git a/public/css/toggle-dependent.css b/public/css/toggle-dependent.css index 30fb33bcf..834e31c31 100644 --- a/public/css/toggle-dependent.css +++ b/public/css/toggle-dependent.css @@ -433,14 +433,6 @@ body.expandMessageActions .mes .mes_buttons .extraMesButtonsHint { display: none !important; } -#openai_image_inlining:not(:checked)~#image_inlining_hint { - display: none; -} - -#openai_image_inlining:checked~#image_inlining_hint { - display: block; -} - #smooth_streaming:not(:checked)~#smooth_streaming_speed_control { display: none; } From bc9c70556e8e3ff8c07aaf7a0aed5dc307c0cecb Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Sun, 21 Apr 2024 23:53:46 +0300 Subject: [PATCH 02/21] Clean-up mentions of /public/ --- .github/workflows/update-docs.yml | 43 ------------------- Update-Instructions.txt | 18 +++++--- .../extensions/expressions/settings.html | 2 +- src/util.js | 6 +-- 4 files changed, 16 insertions(+), 53 deletions(-) delete mode 100644 .github/workflows/update-docs.yml diff --git a/.github/workflows/update-docs.yml b/.github/workflows/update-docs.yml deleted file mode 100644 index 567cac607..000000000 --- a/.github/workflows/update-docs.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Update SillyTavern-Docs - -on: - push: - branches: - - main - -jobs: - update_docs: - runs-on: ubuntu-latest - - steps: - - name: Checkout current repository - uses: actions/checkout@v2 - - - name: Checkout SillyTavern-Docs repository - uses: actions/checkout@v2 - with: - repository: SillyTavern/SillyTavern-Docs - path: SillyTavern-Docs - - - name: Clone SillyTavern wiki into SillyTavern-Docs/extensions - run: rm -rf SillyTavern-Docs/extensions && git clone https://github.com/SillyTavern/SillyTavern.wiki.git SillyTavern-Docs/extensions && rm -rf SillyTavern-Docs/extensions/.git - - - name: Copy files - run: | - cp public/notes/content.md SillyTavern-Docs/guidebook.md - cp faq.md SillyTavern-Docs/faq.md - cp readme.md SillyTavern-Docs/readme.md - cp public/notes/update.md SillyTavern-Docs/update.md - - - name: Deploy to external repository - uses: cpina/github-action-push-to-another-repository@main - env: - SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }} - with: - # GitHub Action output files - source-directory: SillyTavern-Docs/ - destination-github-username: SillyTavern - destination-repository-name: SillyTavern-Docs - user-email: github-actions[bot]@users.noreply.github.com - user-name: "GitHub Actions" - target-branch: "main" diff --git a/Update-Instructions.txt b/Update-Instructions.txt index f153660b2..6e7071184 100644 --- a/Update-Instructions.txt +++ b/Update-Instructions.txt @@ -33,7 +33,14 @@ If you insist on installing via a zip, here is the tedious process for doing the 2. Unzip it into a folder OUTSIDE of your current ST installation. 3. Do the usual setup procedure for your OS to install the NodeJS requirements. -4. Copy the following files/folders as necessary(*) from your old ST installation: +4a. Updating 1.12.0 and above + +Copy the user data directory from your data root into the data root of the new install. + +By default: /data/default-user + +4a. Migrating from <1.12.0 to >=1.20.0 +Copy the following files/folders as necessary(*) from your old ST installation: - Assets - Backgrounds @@ -54,16 +61,15 @@ If you insist on installing via a zip, here is the tedious process for doing the - Worlds - User - settings.json - - secrets.json <---- this one is in the base folder, not /public/ + - secrets.json <---- This one is in the base folder, not /public/ (*) 'As necessary' = "If you made any custom content related to those folders". None of the folders are mandatory, so only copy what you need. **NB: DO NOT COPY THE ENTIRE /PUBLIC/ FOLDER.** Doing so could break the new install and prevent new features from being present. + Paste those items into the /data/default-user folder of the new install. -5. Paste those items into the /Public/ folder of the new install. +5. Start SillyTavern once again with the method appropriate to your OS, and pray you got it right. -6. Start SillyTavern once again with the method appropriate to your OS, and pray you got it right. - -7. If everything shows up, you can safely delete the old ST folder. +6. If everything shows up, you can safely delete the old ST folder. diff --git a/public/scripts/extensions/expressions/settings.html b/public/scripts/extensions/expressions/settings.html index 4a7347a74..e8b1484b2 100644 --- a/public/scripts/extensions/expressions/settings.html +++ b/public/scripts/extensions/expressions/settings.html @@ -78,7 +78,7 @@ Remove all image overrides -

Hint: Create new folder in the public/characters/ folder and name it as the name of the character. +

Hint: Create new folder in the /characters/ folder of your user data directory and name it as the name of the character. Put images with expressions there. File names should follow the pattern: [expression_label].[image_format]

diff --git a/src/util.js b/src/util.js index e1410eee8..ab19f3ccf 100644 --- a/src/util.js +++ b/src/util.js @@ -311,9 +311,9 @@ function tryParse(str) { } /** - * Takes a path to a client-accessible file in the `public` folder and converts it to a relative URL segment that the - * client can fetch it from. This involves stripping the `public/` prefix and always using `/` as the separator. - * @param {string} root The root directory of the public folder. + * Takes a path to a client-accessible file in the data folder and converts it to a relative URL segment that the + * client can fetch it from. This involves stripping the data root path prefix and always using `/` as the separator. + * @param {string} root The root directory of the user data folder. * @param {string} inputPath The path to be converted. * @returns The relative URL path from which the client can access the file. */ From df93d43c36de5ffe4cbb28930c6e68ad05f09388 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 22 Apr 2024 00:02:48 +0300 Subject: [PATCH 03/21] Remove obnoxious mobile padding on right panel --- public/css/mobile-styles.css | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/css/mobile-styles.css b/public/css/mobile-styles.css index 68534d8a2..8456d54a0 100644 --- a/public/css/mobile-styles.css +++ b/public/css/mobile-styles.css @@ -231,9 +231,11 @@ backdrop-filter: blur(calc(var(--SmartThemeBlurStrength) * 2)); } + /* #right-nav-panel { padding-right: 15px; } + */ #floatingPrompt, #cfgConfig, From 41ad7c5d266d33b9432ab94b5cded10ffab29f1a Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 22 Apr 2024 02:34:50 +0300 Subject: [PATCH 04/21] Verify data bank attachments --- public/scripts/chats.js | 52 ++++++++++++++++++++++++++++++++++++++--- src/endpoints/files.js | 25 ++++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/public/scripts/chats.js b/public/scripts/chats.js index a4082f6f0..19e316fa5 100644 --- a/public/scripts/chats.js +++ b/public/scripts/chats.js @@ -592,9 +592,10 @@ async function deleteMessageImage() { /** * Deletes file from the server. * @param {string} url Path to the file on the server + * @param {boolean} [silent=false] If true, do not show error messages * @returns {Promise} True if file was deleted, false otherwise. */ -async function deleteFileFromServer(url) { +async function deleteFileFromServer(url, silent = false) { try { const result = await fetch('/api/files/delete', { method: 'POST', @@ -602,7 +603,7 @@ async function deleteFileFromServer(url) { body: JSON.stringify({ path: url }), }); - if (!result.ok) { + if (!result.ok && !silent) { const error = await result.text(); throw new Error(error); } @@ -702,7 +703,8 @@ async function deleteAttachment(attachment, source, callback, confirm = true) { break; } - await deleteFileFromServer(attachment.url); + const silent = confirm === false; + await deleteFileFromServer(attachment.url, silent); callback(); } @@ -756,6 +758,7 @@ async function openAttachmentManager() { for (const attachment of sortedAttachmentList) { const attachmentTemplate = template.find('.attachmentListItemTemplate .attachmentListItem').clone(); + attachmentTemplate.find('.attachmentFileIcon').attr('title', attachment.url); attachmentTemplate.find('.attachmentListItemName').text(attachment.name); attachmentTemplate.find('.attachmentListItemSize').text(humanFileSize(attachment.size)); attachmentTemplate.find('.attachmentListItemCreated').text(new Date(attachment.created).toLocaleString()); @@ -883,6 +886,7 @@ async function openAttachmentManager() { }); const cleanupFn = await renderButtons(); + await verifyAttachments(); await renderAttachments(); await callGenericPopup(template, POPUP_TYPE.TEXT, '', { wide: true, large: true, okButton: 'Close' }); @@ -1039,6 +1043,48 @@ export function getDataBankAttachmentsForSource(source) { } } +/** + * Verifies all attachments in the Data Bank. + * @returns {Promise} A promise that resolves when attachments are verified. + */ +async function verifyAttachments() { + for (const source of Object.values(ATTACHMENT_SOURCE)) { + await verifyAttachmentsForSource(source); + } +} + +/** + * Verifies all attachments for a specific source. + * @param {string} source Attachment source + * @returns {Promise} A promise that resolves when attachments are verified. + */ +async function verifyAttachmentsForSource(source) { + try { + const attachments = getDataBankAttachmentsForSource(source); + const urls = attachments.map(a => a.url); + const response = await fetch('/api/files/verify', { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ urls }), + }); + + if (!response.ok) { + const error = await response.text(); + throw new Error(error); + } + + const verifiedUrls = await response.json(); + for (const attachment of attachments) { + if (verifiedUrls[attachment.url] === false) { + console.log('Deleting orphaned attachment', attachment); + await deleteAttachment(attachment, source, () => { }, false); + } + } + } catch (error) { + console.error('Attachment verification failed', error); + } +} + /** * Registers a file converter function. * @param {string} mimeType MIME type diff --git a/src/endpoints/files.js b/src/endpoints/files.js index 1c66273bd..371381c21 100644 --- a/src/endpoints/files.js +++ b/src/endpoints/files.js @@ -57,4 +57,29 @@ router.post('/delete', jsonParser, async (request, response) => { } }); +router.post('/verify', jsonParser, async (request, response) => { + try { + if (!Array.isArray(request.body.urls)) { + return response.status(400).send('No URLs specified'); + } + + const verified = {}; + + for (const url of request.body.urls) { + const pathToVerify = path.join(request.user.directories.root, url); + if (!pathToVerify.startsWith(request.user.directories.files)) { + console.debug(`File verification: Invalid path: ${pathToVerify}`); + continue; + } + const fileExists = fs.existsSync(pathToVerify); + verified[url] = fileExists; + } + + return response.send(verified); + } catch (error) { + console.log(error); + return response.sendStatus(500); + } +}); + module.exports = { router }; From 2f45f50d370afc8de4bf3cc2be61c7161e66790c Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 22 Apr 2024 15:52:59 +0300 Subject: [PATCH 05/21] Add config value for forwarded IPs whitelisting --- default/config.yaml | 2 ++ src/middleware/whitelist.js | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/default/config.yaml b/default/config.yaml index 8966447d0..355573d96 100644 --- a/default/config.yaml +++ b/default/config.yaml @@ -9,6 +9,8 @@ port: 8000 # -- SECURITY CONFIGURATION -- # Toggle whitelist mode whitelistMode: true +# Whitelist will also verify IP in X-Forwarded-For / X-Real-IP headers +enableForwardedWhitelist: true # Whitelist of allowed IP addresses whitelist: - 127.0.0.1 diff --git a/src/middleware/whitelist.js b/src/middleware/whitelist.js index def408650..24c1af8e5 100644 --- a/src/middleware/whitelist.js +++ b/src/middleware/whitelist.js @@ -6,6 +6,7 @@ const { getIpFromRequest } = require('../express-common'); const { color, getConfigValue } = require('../util'); const whitelistPath = path.join(process.cwd(), './whitelist.txt'); +const enableForwardedWhitelist = getConfigValue('enableForwardedWhitelist', false); let whitelist = getConfigValue('whitelist', []); let knownIPs = new Set(); @@ -24,14 +25,18 @@ if (fs.existsSync(whitelistPath)) { * @returns {string|undefined} The client IP address */ function getForwardedIp(req) { + if (!enableForwardedWhitelist) { + return undefined; + } + // Check if X-Real-IP is available if (req.headers['x-real-ip']) { - return req.headers['x-real-ip']; + return req.headers['x-real-ip'].toString(); } // Check for X-Forwarded-For and parse if available if (req.headers['x-forwarded-for']) { - const ipList = req.headers['x-forwarded-for'].split(',').map(ip => ip.trim()); + const ipList = req.headers['x-forwarded-for'].toString().split(',').map(ip => ip.trim()); return ipList[0]; } From 5a5463bd5d0123b1d82090ddc08b19db0196f15d Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 22 Apr 2024 16:02:50 +0300 Subject: [PATCH 06/21] #2095 Suppress auto-execution on streamed swiped generations. --- public/scripts/extensions/quick-reply/index.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/public/scripts/extensions/quick-reply/index.js b/public/scripts/extensions/quick-reply/index.js index 7b58f4aaa..74cdbeb78 100644 --- a/public/scripts/extensions/quick-reply/index.js +++ b/public/scripts/extensions/quick-reply/index.js @@ -1,4 +1,4 @@ -import { chat_metadata, eventSource, event_types, getRequestHeaders } from '../../../script.js'; +import { chat, chat_metadata, eventSource, event_types, getRequestHeaders } from '../../../script.js'; import { extension_settings } from '../../extensions.js'; import { QuickReplyApi } from './api/QuickReplyApi.js'; import { AutoExecuteHandler } from './src/AutoExecuteHandler.js'; @@ -238,7 +238,12 @@ const onUserMessage = async () => { }; eventSource.on(event_types.USER_MESSAGE_RENDERED, (...args)=>executeIfReadyElseQueue(onUserMessage, args)); -const onAiMessage = async () => { +const onAiMessage = async (messageId) => { + if (['...'].includes(chat[messageId]?.mes)) { + log('QR auto-execution suppressed for swiped message'); + return; + } + await autoExec.handleAi(); }; eventSource.on(event_types.CHARACTER_MESSAGE_RENDERED, (...args)=>executeIfReadyElseQueue(onAiMessage, args)); From 776260c85ad4c7d731e9f3059f868fba3e6106da Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 22 Apr 2024 16:25:46 +0300 Subject: [PATCH 07/21] Add Data Bank to attachments extension display name --- public/scripts/extensions/attachments/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/scripts/extensions/attachments/manifest.json b/public/scripts/extensions/attachments/manifest.json index 2037168c2..27f55f77c 100644 --- a/public/scripts/extensions/attachments/manifest.json +++ b/public/scripts/extensions/attachments/manifest.json @@ -1,5 +1,5 @@ { - "display_name": "Chat Attachments", + "display_name": "Data Bank (Chat Attachments)", "loading_order": 3, "requires": [], "optional": [], From 6d1933c8f35c7702cb04a1904b1e12f9a3198d05 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 22 Apr 2024 17:35:42 +0300 Subject: [PATCH 08/21] Escape name regex in message formatting function --- public/script.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/script.js b/public/script.js index 9bbb8dfa0..3f28b2e7b 100644 --- a/public/script.js +++ b/public/script.js @@ -1840,7 +1840,7 @@ function messageFormatting(mes, ch_name, isSystem, isUser, messageId) { */ if (!power_user.allow_name2_display && ch_name && !isUser && !isSystem) { - mes = mes.replace(new RegExp(`(^|\n)${ch_name}:`, 'g'), '$1'); + mes = mes.replace(new RegExp(`(^|\n)${escapeRegex(ch_name)}:`, 'g'), '$1'); } /** @type {any} */ From 4370db6bdc7f6959caa378c27add3d0c16508eb8 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 23 Apr 2024 03:09:52 +0300 Subject: [PATCH 09/21] Implement World Info activation using Vector Storage --- public/index.html | 11 +- public/script.js | 1 + public/scripts/extensions/vectors/index.js | 135 ++++++++++++++++++ .../scripts/extensions/vectors/settings.html | 40 ++++++ public/scripts/world-info.js | 77 ++++++++-- src/endpoints/characters.js | 1 + 6 files changed, 250 insertions(+), 15 deletions(-) diff --git a/public/index.html b/public/index.html index ab7329d76..75aaf4eac 100644 --- a/public/index.html +++ b/public/index.html @@ -4955,10 +4955,11 @@ - + + + +
@@ -6065,4 +6066,4 @@ - \ No newline at end of file + diff --git a/public/script.js b/public/script.js index 3f28b2e7b..a1df1eeb2 100644 --- a/public/script.js +++ b/public/script.js @@ -449,6 +449,7 @@ export const event_types = { CHARACTER_DUPLICATED: 'character_duplicated', SMOOTH_STREAM_TOKEN_RECEIVED: 'smooth_stream_token_received', FILE_ATTACHMENT_DELETED: 'file_attachment_deleted', + WORLDINFO_FORCE_ACTIVATE: 'worldinfo_force_activate', }; export const eventSource = new EventEmitter(); diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index 6ff5c6af5..5cdd0830b 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -23,6 +23,7 @@ import { collapseNewlines } from '../../power-user.js'; import { SECRET_KEYS, secret_state, writeSecret } from '../../secrets.js'; import { getDataBankAttachments, getFileAttachment } from '../../chats.js'; import { debounce, getStringHash as calculateHash, waitUntilCondition, onlyUnique, splitRecursive } from '../../utils.js'; +import { getSortedEntries } from '../../world-info.js'; const MODULE_NAME = 'vectors'; @@ -66,6 +67,11 @@ const settings = { file_position_db: extension_prompt_types.IN_PROMPT, file_depth_db: 4, file_depth_role_db: extension_prompt_roles.SYSTEM, + + // For World Info + enabled_world_info: false, + enabled_for_all: false, + max_entries: 5, }; const moduleWorker = new ModuleWorkerWrapper(synchronizeChat); @@ -472,6 +478,10 @@ async function rearrangeChat(chat) { await processFiles(chat); } + if (settings.enabled_world_info) { + await activateWorldInfo(chat); + } + if (!settings.enabled_chats) { return; } @@ -845,6 +855,7 @@ async function purgeVectorIndex(collectionId) { function toggleSettings() { $('#vectors_files_settings').toggle(!!settings.enabled_files); $('#vectors_chats_settings').toggle(!!settings.enabled_chats); + $('#vectors_world_info_settings').toggle(!!settings.enabled_world_info); $('#together_vectorsModel').toggle(settings.source === 'togetherai'); $('#openai_vectorsModel').toggle(settings.source === 'openai'); $('#cohere_vectorsModel').toggle(settings.source === 'cohere'); @@ -934,6 +945,111 @@ async function onPurgeFilesClick() { } } +async function activateWorldInfo(chat) { + if (!settings.enabled_world_info) { + console.debug('Vectors: Disabled for World Info'); + return; + } + + const entries = await getSortedEntries(); + + if (!Array.isArray(entries) || entries.length === 0) { + console.debug('Vectors: No WI entries found'); + return; + } + + // Group entries by "world" field + const groupedEntries = {}; + + for (const entry of entries) { + // Skip orphaned entries. Is it even possible? + if (!entry.world) { + console.debug('Vectors: Skipped orphaned WI entry', entry); + continue; + } + + // Skip disabled entries + if (entry.disable) { + console.debug('Vectors: Skipped disabled WI entry', entry); + continue; + } + + // Skip entries without content + if (!entry.content) { + console.debug('Vectors: Skipped WI entry without content', entry); + continue; + } + + // Skip non-vectorized entries + if (!entry.vectorized && !settings.enabled_for_all) { + console.debug('Vectors: Skipped non-vectorized WI entry', entry); + continue; + } + + if (!Object.hasOwn(groupedEntries, entry.world)) { + groupedEntries[entry.world] = []; + } + + groupedEntries[entry.world].push(entry); + } + + const collectionIds = []; + + if (Object.keys(groupedEntries).length === 0) { + console.debug('Vectors: No WI entries to synchronize'); + return; + } + + // Synchronize collections + for (const world in groupedEntries) { + const collectionId = `world_${getStringHash(world)}`; + const hashesInCollection = await getSavedHashes(collectionId); + const newEntries = groupedEntries[world].filter(x => !hashesInCollection.includes(getStringHash(x.content))); + const deletedHashes = hashesInCollection.filter(x => !groupedEntries[world].some(y => getStringHash(y.content) === x)); + + if (newEntries.length > 0) { + console.log(`Vectors: Found ${newEntries.length} new WI entries for world ${world}`); + await insertVectorItems(collectionId, newEntries.map(x => ({ hash: getStringHash(x.content), text: x.content, index: x.uid }))); + } + + if (deletedHashes.length > 0) { + console.log(`Vectors: Deleted ${deletedHashes.length} old hashes for world ${world}`); + await deleteVectorItems(collectionId, deletedHashes); + } + + collectionIds.push(collectionId); + } + + // Perform a multi-query + const queryText = await getQueryText(chat); + + if (queryText.length === 0) { + console.debug('Vectors: No text to query for WI'); + return; + } + + const queryResults = await queryMultipleCollections(collectionIds, queryText, settings.max_entries); + const activatedHashes = Object.values(queryResults).flatMap(x => x.hashes).filter(onlyUnique); + const activatedEntries = []; + + // Activate entries found in the query results + for (const entry of entries) { + const hash = getStringHash(entry.content); + + if (activatedHashes.includes(hash)) { + activatedEntries.push(entry); + } + } + + if (activatedEntries.length === 0) { + console.debug('Vectors: No activated WI entries found'); + return; + } + + console.log(`Vectors: Activated ${activatedEntries.length} WI entries`, activatedEntries); + await eventSource.emit(event_types.WORLDINFO_FORCE_ACTIVATE, activatedEntries); +} + jQuery(async () => { if (!extension_settings.vectors) { extension_settings.vectors = settings; @@ -1134,6 +1250,25 @@ jQuery(async () => { saveSettingsDebounced(); }); + $('#vectors_enabled_world_info').prop('checked', settings.enabled_world_info).on('input', () => { + settings.enabled_world_info = !!$('#vectors_enabled_world_info').prop('checked'); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + toggleSettings(); + }); + + $('#vectors_enabled_for_all').prop('checked', settings.enabled_for_all).on('input', () => { + settings.enabled_for_all = !!$('#vectors_enabled_for_all').prop('checked'); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + }); + + $('#vectors_max_entries').val(settings.max_entries).on('input', () => { + settings.max_entries = Number($('#vectors_max_entries').val()); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + }); + const validSecret = !!secret_state[SECRET_KEYS.NOMICAI]; const placeholder = validSecret ? '✔️ Key saved' : '❌ Missing key'; $('#api_key_nomicai').attr('placeholder', placeholder); diff --git a/public/scripts/extensions/vectors/settings.html b/public/scripts/extensions/vectors/settings.html index 02499d120..bcaf1c06e 100644 --- a/public/scripts/extensions/vectors/settings.html +++ b/public/scripts/extensions/vectors/settings.html @@ -97,6 +97,46 @@
+

+ World Info settings +

+ + + +
+
+ +
    +
  • + Checked: all entries except ❌ status can be activated. +
    • +
  • + Unchecked: only entries with 🔗 status can be activated. +
    • +
+
+
+
+ +
+
+ + +
+
+ +
+
+
+

File vectorization settings

diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index 2932e9050..767f923d0 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -82,6 +82,11 @@ class WorldInfoBuffer { /** @typedef {{scanDepth?: number, caseSensitive?: boolean, matchWholeWords?: boolean}} WIScanEntry The entry that triggered the scan */ // End typedef area + /** + * @type {object[]} Array of entries that need to be activated no matter what + */ + static externalActivations = []; + /** * @type {string[]} Array of messages sorted by ascending depth */ @@ -220,6 +225,23 @@ class WorldInfoBuffer { getDepth() { return world_info_depth + this.#skew; } + + /** + * Check if the current entry is externally activated. + * @param {object} entry WI entry to check + * @returns {boolean} True if the entry is forcefully activated + */ + isExternallyActivated(entry) { + // Entries could be copied with structuredClone, so we need to compare them by string representation + return WorldInfoBuffer.externalActivations.some(x => JSON.stringify(x) === JSON.stringify(entry)); + } + + /** + * Clears the force activations buffer. + */ + cleanExternalActivations() { + WorldInfoBuffer.externalActivations.splice(0, WorldInfoBuffer.externalActivations.length); + } } export function getWorldInfoSettings() { @@ -362,6 +384,10 @@ function setWorldInfoSettings(settings, data) { $('.chat_lorebook_button').toggleClass('world_set', hasWorldInfo); }); + eventSource.on(event_types.WORLDINFO_FORCE_ACTIVATE, (entries) => { + WorldInfoBuffer.externalActivations.push(...entries); + }); + // Add slash commands registerWorldInfoSlashCommands(); } @@ -564,6 +590,7 @@ function registerWorldInfoSlashCommands() { return ''; } + registerSlashCommand('world', onWorldInfoChange, [], '[optional state=off|toggle] [optional silent=true] (optional name) – sets active World, or unsets if no args provided, use state=off and state=toggle to deactivate or toggle a World, use silent=true to suppress toast messages', true, true); registerSlashCommand('getchatbook', getChatBookCallback, ['getchatlore', 'getchatwi'], '– get a name of the chat-bound lorebook or create a new one if was unbound, and pass it down the pipe', true, true); registerSlashCommand('findentry', findBookEntryCallback, ['findlore', 'findwi'], '(file=bookName field=field [texts]) – find a UID of the record from the specified book using the fuzzy match of a field value (default: key) and pass it down the pipe, e.g. /findentry file=chatLore field=key Shadowfang', true, true); registerSlashCommand('getentryfield', getEntryFieldCallback, ['getlorefield', 'getwifield'], '(file=bookName field=field [UID]) – get a field value (default: content) of the record with the UID from the specified book and pass it down the pipe, e.g. /getentryfield file=chatLore field=content 123', true, true); @@ -964,6 +991,7 @@ const originalDataKeyMap = { 'caseSensitive': 'extensions.case_sensitive', 'scanDepth': 'extensions.scan_depth', 'automationId': 'extensions.automation_id', + 'vectorized': 'extensions.vectorized', }; function setOriginalDataValue(data, uid, key, value) { @@ -1071,6 +1099,16 @@ function getWorldEntry(name, data, entry) { ); } + // Verify names to exist in the system + if (data.entries[uid]?.characterFilter?.names?.length > 0) { + for (const name of [...data.entries[uid].characterFilter.names]) { + if (!getContext().characters.find(x => x.avatar.replace(/\.[^/.]+$/, '') === name)) { + console.warn(`World Info: Character ${name} not found. Removing from the entry filter.`, entry); + data.entries[uid].characterFilter.names = data.entries[uid].characterFilter.names.filter(x => x !== name); + } + } + } + setOriginalDataValue(data, uid, 'character_filter', data.entries[uid].characterFilter); saveWorldInfo(name, data); }); @@ -1454,22 +1492,37 @@ function getWorldEntry(name, data, entry) { case 'constant': data.entries[uid].constant = true; data.entries[uid].disable = false; + data.entries[uid].vectorized = false; setOriginalDataValue(data, uid, 'enabled', true); setOriginalDataValue(data, uid, 'constant', true); + setOriginalDataValue(data, uid, 'extensions.vectorized', false); template.removeClass('disabledWIEntry'); break; case 'normal': data.entries[uid].constant = false; data.entries[uid].disable = false; + data.entries[uid].vectorized = false; setOriginalDataValue(data, uid, 'enabled', true); setOriginalDataValue(data, uid, 'constant', false); + setOriginalDataValue(data, uid, 'extensions.vectorized', false); + template.removeClass('disabledWIEntry'); + break; + case 'vectorized': + data.entries[uid].constant = false; + data.entries[uid].disable = false; + data.entries[uid].vectorized = true; + setOriginalDataValue(data, uid, 'enabled', true); + setOriginalDataValue(data, uid, 'constant', false); + setOriginalDataValue(data, uid, 'extensions.vectorized', true); template.removeClass('disabledWIEntry'); break; case 'disabled': data.entries[uid].constant = false; data.entries[uid].disable = true; + data.entries[uid].vectorized = false; setOriginalDataValue(data, uid, 'enabled', false); setOriginalDataValue(data, uid, 'constant', false); + setOriginalDataValue(data, uid, 'extensions.vectorized', false); template.addClass('disabledWIEntry'); break; } @@ -1480,6 +1533,8 @@ function getWorldEntry(name, data, entry) { const entryState = function () { if (entry.constant === true) { return 'constant'; + } else if (entry.vectorized === true) { + return 'vectorized'; } else if (entry.disable === true) { return 'disabled'; } else { @@ -1719,6 +1774,7 @@ const newEntryTemplate = { comment: '', content: '', constant: false, + vectorized: false, selective: true, selectiveLogic: world_info_logic.AND_ANY, addMemo: false, @@ -1925,7 +1981,7 @@ async function getCharacterLore() { } const data = await loadWorldInfoData(worldName); - const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]) : []; + const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x => ({ ...x, world: worldName })) : []; entries = entries.concat(newEntries); } @@ -1941,7 +1997,7 @@ async function getGlobalLore() { let entries = []; for (const worldName of selected_world_info) { const data = await loadWorldInfoData(worldName); - const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]) : []; + const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x => ({ ...x, world: worldName })) : []; entries = entries.concat(newEntries); } @@ -1963,14 +2019,14 @@ async function getChatLore() { } const data = await loadWorldInfoData(chatWorld); - const entries = data ? Object.keys(data.entries).map((x) => data.entries[x]) : []; + const entries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x => ({ ...x, world: chatWorld })) : []; console.debug(`Chat lore has ${entries.length} entries`); return entries; } -async function getSortedEntries() { +export async function getSortedEntries() { try { const globalLore = await getGlobalLore(); const characterLore = await getCharacterLore(); @@ -2098,7 +2154,7 @@ async function checkWorldInfo(chat, maxContext) { continue; } - if (entry.constant) { + if (entry.constant || buffer.isExternallyActivated(entry)) { entry.content = substituteParams(entry.content); activatedNow.add(entry); continue; @@ -2295,6 +2351,8 @@ async function checkWorldInfo(chat, maxContext) { context.setExtensionPrompt(NOTE_MODULE_NAME, ANWithWI, chat_metadata[metadata_keys.position], chat_metadata[metadata_keys.depth], extension_settings.note.allowWIScan, chat_metadata[metadata_keys.role]); } + buffer.cleanExternalActivations(); + return { worldInfoBefore, worldInfoAfter, WIDepthEntries, allActivatedEntries }; } @@ -2381,6 +2439,7 @@ function convertAgnaiMemoryBook(inputObj) { content: entry.entry, constant: false, selective: false, + vectorized: false, selectiveLogic: world_info_logic.AND_ANY, order: entry.weight, position: 0, @@ -2415,6 +2474,7 @@ function convertRisuLorebook(inputObj) { content: entry.content, constant: entry.alwaysActive, selective: entry.selective, + vectorized: false, selectiveLogic: world_info_logic.AND_ANY, order: entry.insertorder, position: world_info_position.before, @@ -2454,6 +2514,7 @@ function convertNovelLorebook(inputObj) { content: entry.text, constant: false, selective: false, + vectorized: false, selectiveLogic: world_info_logic.AND_ANY, order: entry.contextConfig?.budgetPriority ?? 0, position: 0, @@ -2510,6 +2571,7 @@ function convertCharacterBook(characterBook) { matchWholeWords: entry.extensions?.match_whole_words ?? null, automationId: entry.extensions?.automation_id ?? '', role: entry.extensions?.role ?? extension_prompt_roles.SYSTEM, + vectorized: entry.extensions?.vectorized ?? false, }; }); @@ -2785,11 +2847,6 @@ function assignLorebookToChat() { jQuery(() => { - $(document).ready(function () { - registerSlashCommand('world', onWorldInfoChange, [], '[optional state=off|toggle] [optional silent=true] (optional name) – sets active World, or unsets if no args provided, use state=off and state=toggle to deactivate or toggle a World, use silent=true to suppress toast messages', true, true); - }); - - $('#world_info').on('mousedown change', async function (e) { // If there's no world names, don't do anything if (world_names.length === 0) { diff --git a/src/endpoints/characters.js b/src/endpoints/characters.js index 556a8fecc..571eec8c0 100644 --- a/src/endpoints/characters.js +++ b/src/endpoints/characters.js @@ -439,6 +439,7 @@ function convertWorldInfoToCharacterBook(name, entries) { case_sensitive: entry.caseSensitive ?? null, automation_id: entry.automationId ?? '', role: entry.role ?? 0, + vectorized: entry.vectorized ?? false, }, }; From d97f0a4c4d875d09591b8f4e63251020c3c3da5d Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 23 Apr 2024 03:18:45 +0300 Subject: [PATCH 10/21] Add new NAI Diffusion model --- public/scripts/extensions/stable-diffusion/index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js index d7b0cfc0b..88fdbad40 100644 --- a/public/scripts/extensions/stable-diffusion/index.js +++ b/public/scripts/extensions/stable-diffusion/index.js @@ -1657,6 +1657,10 @@ async function loadNovelModels() { value: 'safe-diffusion', text: 'NAI Diffusion Anime V1 (Curated)', }, + { + value: 'nai-diffusion-furry-3', + text: 'NAI Diffusion Furry V3', + }, { value: 'nai-diffusion-furry', text: 'NAI Diffusion Furry', From 890cf8162781ed1e007dc86e506cc4cfb55387eb Mon Sep 17 00:00:00 2001 From: joenunezb Date: Tue, 23 Apr 2024 03:56:50 -0700 Subject: [PATCH 11/21] Fix: InformaticAI response without message in choices --- src/endpoints/backends/text-completions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js index 22806cbf0..0e9598827 100644 --- a/src/endpoints/backends/text-completions.js +++ b/src/endpoints/backends/text-completions.js @@ -325,7 +325,7 @@ router.post('/generate', jsonParser, async function (request, response) { // Map InfermaticAI response to OAI completions format if (apiType === TEXTGEN_TYPES.INFERMATICAI) { - data['choices'] = (data?.choices || []).map(choice => ({ text: choice.message.content })); + data['choices'] = (data?.choices || []).map(choice => ({ text: choice?.message?.content || choice.text })); } return response.send(data); From 75372ad0ccbfcc3b60010e3177ebd27a2afd8835 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 23 Apr 2024 16:15:54 +0300 Subject: [PATCH 12/21] Use Map for caches instead of objects --- public/scripts/extensions/vectors/index.js | 12 +++++++----- src/endpoints/classify.js | 16 ++++++++++++---- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index 5cdd0830b..ca2ddbdeb 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -287,8 +287,10 @@ async function synchronizeChat(batchSize = 5) { } } -// Cache object for storing hash values -const hashCache = {}; +/** + * @type {Map} Cache object for storing hash values + */ +const hashCache = new Map(); /** * Gets the hash value for a given string @@ -297,15 +299,15 @@ const hashCache = {}; */ function getStringHash(str) { // Check if the hash is already in the cache - if (Object.hasOwn(hashCache, str)) { - return hashCache[str]; + if (hashCache.has(str)) { + return hashCache.get(str); } // Calculate the hash value const hash = calculateHash(str); // Store the hash in the cache - hashCache[str] = hash; + hashCache.set(str, hash); return hash; } diff --git a/src/endpoints/classify.js b/src/endpoints/classify.js index 5a9772e1d..758b95247 100644 --- a/src/endpoints/classify.js +++ b/src/endpoints/classify.js @@ -5,7 +5,10 @@ const TASK = 'text-classification'; const router = express.Router(); -const cacheObject = {}; +/** + * @type {Map} Cache for classification results + */ +const cacheObject = new Map(); router.post('/labels', jsonParser, async (req, res) => { try { @@ -23,15 +26,20 @@ router.post('/', jsonParser, async (req, res) => { try { const { text } = req.body; + /** + * Get classification result for a given text + * @param {string} text Text to classify + * @returns {Promise} Classification result + */ async function getResult(text) { - if (Object.hasOwn(cacheObject, text)) { - return cacheObject[text]; + if (cacheObject.has(text)) { + return cacheObject.get(text); } else { const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(TASK); const result = await pipe(text, { topk: 5 }); result.sort((a, b) => b.score - a.score); - cacheObject[text] = result; + cacheObject.set(text, result); return result; } } From a421af9ea9c8a2b9246feb37cdff5fc79d63faba Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 23 Apr 2024 21:06:59 +0300 Subject: [PATCH 13/21] Increase max attachment size --- public/scripts/chats.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/scripts/chats.js b/public/scripts/chats.js index 19e316fa5..f5fa6a851 100644 --- a/public/scripts/chats.js +++ b/public/scripts/chats.js @@ -53,7 +53,7 @@ import { ScraperManager } from './scrapers.js'; * @returns {Promise} Converted file text */ -const fileSizeLimit = 1024 * 1024 * 10; // 10 MB +const fileSizeLimit = 1024 * 1024 * 100; // 100 MB const ATTACHMENT_SOURCE = { GLOBAL: 'global', CHAT: 'chat', From 71f41d52330f5b4e9993e00e01803638d500688b Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Tue, 23 Apr 2024 21:11:47 +0300 Subject: [PATCH 14/21] Fix server crash in auto login --- src/users.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/users.js b/src/users.js index 831ff09d5..023ddca07 100644 --- a/src/users.js +++ b/src/users.js @@ -511,7 +511,7 @@ async function tryAutoLogin(request) { const userHandles = await getAllUserHandles(); if (userHandles.length === 1) { const user = await storage.getItem(toKey(userHandles[0])); - if (!user.password) { + if (user && !user.password) { request.session.handle = userHandles[0]; return true; } From b6b9b542d7a10f8a129321967db24b3dbabd1366 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Wed, 24 Apr 2024 01:51:54 +0300 Subject: [PATCH 15/21] Add drag&drop to data bank --- public/scripts/chats.js | 57 +++++++++++++++++++ .../extensions/attachments/files-dropped.html | 8 +++ public/style.css | 7 ++- 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 public/scripts/extensions/attachments/files-dropped.html diff --git a/public/scripts/chats.js b/public/scripts/chats.js index f5fa6a851..778ee7b6a 100644 --- a/public/scripts/chats.js +++ b/public/scripts/chats.js @@ -863,6 +863,61 @@ async function openAttachmentManager() { template.find('.chatAttachmentsName').text(chatName); } + function addDragAndDrop() { + $(document.body).on('dragover', '.dialogue_popup', (event) => { + event.preventDefault(); + event.stopPropagation(); + $(event.target).closest('.dialogue_popup').addClass('dragover'); + }); + + $(document.body).on('dragleave', '.dialogue_popup', (event) => { + event.preventDefault(); + event.stopPropagation(); + $(event.target).closest('.dialogue_popup').removeClass('dragover'); + }); + + $(document.body).on('drop', '.dialogue_popup', async (event) => { + event.preventDefault(); + event.stopPropagation(); + $(event.target).closest('.dialogue_popup').removeClass('dragover'); + + const files = Array.from(event.originalEvent.dataTransfer.files); + const targets = Object.values(ATTACHMENT_SOURCE); + + const isNotCharacter = this_chid === undefined || selected_group; + const isNotInChat = getCurrentChatId() === undefined; + let selectedTarget = ATTACHMENT_SOURCE.GLOBAL; + + if (isNotCharacter) { + targets.splice(targets.indexOf(ATTACHMENT_SOURCE.CHARACTER), 1); + } + + if (isNotInChat) { + targets.splice(targets.indexOf(ATTACHMENT_SOURCE.CHAT), 1); + } + + const targetSelectTemplate = $(await renderExtensionTemplateAsync('attachments', 'files-dropped', { count: files.length, targets: targets })); + targetSelectTemplate.find('.droppedFilesTarget').on('input', function () { + selectedTarget = String($(this).val()); + }); + const result = await callGenericPopup(targetSelectTemplate, POPUP_TYPE.CONFIRM, '', { wide: false, large: false, okButton: 'Upload', cancelButton: 'Cancel' }); + if (result !== POPUP_RESULT.AFFIRMATIVE) { + console.log('File upload cancelled'); + return; + } + for (const file of files) { + await uploadFileAttachmentToServer(file, selectedTarget); + } + renderAttachments(); + }); + } + + function removeDragAndDrop() { + $(document.body).off('dragover', '.shadow_popup'); + $(document.body).off('dragleave', '.shadow_popup'); + $(document.body).off('drop', '.shadow_popup'); + } + let sortField = localStorage.getItem('DataBank_sortField') || 'created'; let sortOrder = localStorage.getItem('DataBank_sortOrder') || 'desc'; let filterString = ''; @@ -888,9 +943,11 @@ async function openAttachmentManager() { const cleanupFn = await renderButtons(); await verifyAttachments(); await renderAttachments(); + addDragAndDrop(); await callGenericPopup(template, POPUP_TYPE.TEXT, '', { wide: true, large: true, okButton: 'Close' }); cleanupFn(); + removeDragAndDrop(); } /** diff --git a/public/scripts/extensions/attachments/files-dropped.html b/public/scripts/extensions/attachments/files-dropped.html new file mode 100644 index 000000000..7295c4994 --- /dev/null +++ b/public/scripts/extensions/attachments/files-dropped.html @@ -0,0 +1,8 @@ +
+ Save {{count}} file(s) to... + +
diff --git a/public/style.css b/public/style.css index 1535c0b4d..9cde87423 100644 --- a/public/style.css +++ b/public/style.css @@ -2262,6 +2262,11 @@ grammarly-extension { } } +.dialogue_popup.dragover { + filter: brightness(1.1) saturate(1.1); + outline: 3px dashed var(--SmartThemeBorderColor); +} + #bgtest { display: none; width: 100vw; @@ -4015,4 +4020,4 @@ body:not(.movingUI) .drawer-content.maximized { height: 100vh; z-index: 9999; } -} \ No newline at end of file +} From 61241df0d4a7025b5cca8515245937c0d260fcd5 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Wed, 24 Apr 2024 02:33:16 +0300 Subject: [PATCH 16/21] Add download and move for DB attachments --- public/scripts/chats.js | 87 ++++++++++++++++--- .../extensions/attachments/files-dropped.html | 2 +- .../extensions/attachments/manager.html | 2 + .../attachments/move-attachment.html | 8 ++ 4 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 public/scripts/extensions/attachments/move-attachment.html diff --git a/public/scripts/chats.js b/public/scripts/chats.js index 778ee7b6a..0d2a9706a 100644 --- a/public/scripts/chats.js +++ b/public/scripts/chats.js @@ -56,8 +56,8 @@ import { ScraperManager } from './scrapers.js'; const fileSizeLimit = 1024 * 1024 * 100; // 100 MB const ATTACHMENT_SOURCE = { GLOBAL: 'global', - CHAT: 'chat', CHARACTER: 'character', + CHAT: 'chat', }; /** @@ -670,6 +670,55 @@ async function editAttachment(attachment, source, callback) { callback(); } +/** + * Downloads an attachment to the user's device. + * @param {FileAttachment} attachment Attachment to download + */ +async function downloadAttachment(attachment) { + const fileText = attachment.text || (await getFileAttachment(attachment.url)); + const blob = new Blob([fileText], { type: 'text/plain' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = attachment.name; + a.click(); + URL.revokeObjectURL(url); +} + +/** + * Moves a file attachment to a different source. + * @param {FileAttachment} attachment Attachment to moves + * @param {string} source Source of the attachment + * @param {function} callback Success callback + * @returns {Promise} A promise that resolves when the attachment is moved. + */ +async function moveAttachment(attachment, source, callback) { + let selectedTarget = source; + const targets = getAvailableTargets(); + const template = $(await renderExtensionTemplateAsync('attachments', 'move-attachment', { name: attachment.name, targets })); + template.find('.moveAttachmentTarget').val(source).on('input', function () { + selectedTarget = String($(this).val()); + }); + + const result = await callGenericPopup(template, POPUP_TYPE.CONFIRM, '', { wide: false, large: false, okButton: 'Move', cancelButton: 'Cancel' }); + + if (result !== POPUP_RESULT.AFFIRMATIVE) { + console.debug('Move attachment cancelled'); + return; + } + + if (selectedTarget === source) { + console.debug('Move attachment cancelled: same source and target'); + return; + } + + const content = await getFileAttachment(attachment.url); + const file = new File([content], attachment.name, { type: 'text/plain' }); + await deleteAttachment(attachment, source, () => { }, false); + await uploadFileAttachmentToServer(file, selectedTarget); + callback(); +} + /** * Deletes an attachment from the server and the chat. * @param {FileAttachment} attachment Attachment to delete @@ -765,6 +814,8 @@ async function openAttachmentManager() { attachmentTemplate.find('.viewAttachmentButton').on('click', () => openFilePopup(attachment)); attachmentTemplate.find('.editAttachmentButton').on('click', () => editAttachment(attachment, source, renderAttachments)); attachmentTemplate.find('.deleteAttachmentButton').on('click', () => deleteAttachment(attachment, source, renderAttachments)); + attachmentTemplate.find('.downloadAttachmentButton').on('click', () => downloadAttachment(attachment)); + attachmentTemplate.find('.moveAttachmentButton').on('click', () => moveAttachment(attachment, source, renderAttachments)); template.find(sources[source]).append(attachmentTemplate); } } @@ -882,19 +933,8 @@ async function openAttachmentManager() { $(event.target).closest('.dialogue_popup').removeClass('dragover'); const files = Array.from(event.originalEvent.dataTransfer.files); - const targets = Object.values(ATTACHMENT_SOURCE); - - const isNotCharacter = this_chid === undefined || selected_group; - const isNotInChat = getCurrentChatId() === undefined; let selectedTarget = ATTACHMENT_SOURCE.GLOBAL; - - if (isNotCharacter) { - targets.splice(targets.indexOf(ATTACHMENT_SOURCE.CHARACTER), 1); - } - - if (isNotInChat) { - targets.splice(targets.indexOf(ATTACHMENT_SOURCE.CHAT), 1); - } + const targets = getAvailableTargets(); const targetSelectTemplate = $(await renderExtensionTemplateAsync('attachments', 'files-dropped', { count: files.length, targets: targets })); targetSelectTemplate.find('.droppedFilesTarget').on('input', function () { @@ -950,6 +990,27 @@ async function openAttachmentManager() { removeDragAndDrop(); } +/** + * Gets a list of available targets for attachments. + * @returns {string[]} List of available targets + */ +function getAvailableTargets() { + const targets = Object.values(ATTACHMENT_SOURCE); + + const isNotCharacter = this_chid === undefined || selected_group; + const isNotInChat = getCurrentChatId() === undefined; + + if (isNotCharacter) { + targets.splice(targets.indexOf(ATTACHMENT_SOURCE.CHARACTER), 1); + } + + if (isNotInChat) { + targets.splice(targets.indexOf(ATTACHMENT_SOURCE.CHAT), 1); + } + + return targets; +} + /** * Runs a known scraper on a source and saves the result as an attachment. * @param {string} scraperId Id of the scraper diff --git a/public/scripts/extensions/attachments/files-dropped.html b/public/scripts/extensions/attachments/files-dropped.html index 7295c4994..9fd014ded 100644 --- a/public/scripts/extensions/attachments/files-dropped.html +++ b/public/scripts/extensions/attachments/files-dropped.html @@ -1,4 +1,4 @@ -
+
Save {{count}} file(s) to... + {{#each targets}} + + {{/each}} + +
From 2bba186c9efa420df544fec0422a9d24c04059e5 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Wed, 24 Apr 2024 02:37:57 +0300 Subject: [PATCH 17/21] Add slash command and d&d hint for data bank --- public/scripts/extensions/attachments/index.js | 3 +++ public/scripts/extensions/attachments/manager.html | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/public/scripts/extensions/attachments/index.js b/public/scripts/extensions/attachments/index.js index a7f58dbc7..ab3db9419 100644 --- a/public/scripts/extensions/attachments/index.js +++ b/public/scripts/extensions/attachments/index.js @@ -1,6 +1,9 @@ import { renderExtensionTemplateAsync } from '../../extensions.js'; +import { registerSlashCommand } from '../../slash-commands.js'; jQuery(async () => { const buttons = await renderExtensionTemplateAsync('attachments', 'buttons', {}); $('#extensionsMenu').prepend(buttons); + + registerSlashCommand('db', () => document.getElementById('manageAttachments')?.click(), ['databank', 'data-bank'], '– open the data bank', true, true); }); diff --git a/public/scripts/extensions/attachments/manager.html b/public/scripts/extensions/attachments/manager.html index 8ad45a98e..68ad732a2 100644 --- a/public/scripts/extensions/attachments/manager.html +++ b/public/scripts/extensions/attachments/manager.html @@ -7,8 +7,13 @@
These files will be available for extensions that support attachments (e.g. Vector Storage).
-
- Supported file types: Plain Text, PDF, Markdown, HTML, EPUB. +
+ + Supported file types: Plain Text, PDF, Markdown, HTML, EPUB. + + + Drag and drop files here to upload. +
From 530bf81940d0d0ad4d082b701736f9bb235ac4fa Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Wed, 24 Apr 2024 10:48:08 +0300 Subject: [PATCH 18/21] #2127 Encode export PNG name --- src/endpoints/characters.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/endpoints/characters.js b/src/endpoints/characters.js index 571eec8c0..929818ade 100644 --- a/src/endpoints/characters.js +++ b/src/endpoints/characters.js @@ -1098,7 +1098,7 @@ router.post('/export', jsonParser, async function (request, response) { const fileContent = await fsPromises.readFile(filename); const contentType = mime.lookup(filename) || 'image/png'; response.setHeader('Content-Type', contentType); - response.setHeader('Content-Disposition', `attachment; filename=${path.basename(filename)}`); + response.setHeader('Content-Disposition', `attachment; filename="${encodeURI(path.basename(filename))}"`); return response.send(fileContent); } case 'json': { From 51014e7a8d2aa40f633dbe8fab8995ad2ae49f5e Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Wed, 24 Apr 2024 10:54:55 +0300 Subject: [PATCH 19/21] Fix VRM assets console spam --- src/endpoints/assets.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/endpoints/assets.js b/src/endpoints/assets.js index 78f5270a7..09cc1aa71 100644 --- a/src/endpoints/assets.js +++ b/src/endpoints/assets.js @@ -56,6 +56,8 @@ function validateAssetFileName(inputFilename) { * @returns {string[]} - The array of files */ function getFiles(dir, files = []) { + if (!fs.existsSync(dir)) return files; + // Get an array of all files and directories in the passed directory using fs.readdirSync const fileList = fs.readdirSync(dir, { withFileTypes: true }); // Create the full path of the file/directory by concatenating the passed directory and file/directory name From 153638c2cd8ee287434a984a56c7d808f257c284 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Wed, 24 Apr 2024 00:59:55 +0300 Subject: [PATCH 20/21] Add error handling to auto login --- server.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/server.js b/server.js index c75304f36..cbb74d900 100644 --- a/server.js +++ b/server.js @@ -263,10 +263,14 @@ app.get('/login', async (request, response) => { return response.redirect('/'); } - const autoLogin = await userModule.tryAutoLogin(request); + try { + const autoLogin = await userModule.tryAutoLogin(request); - if (autoLogin) { - return response.redirect('/'); + if (autoLogin) { + return response.redirect('/'); + } + } catch (error) { + console.error('Error during auto-login:', error); } return response.sendFile('login.html', { root: path.join(process.cwd(), 'public') }); From 01e3964232604953879445a1e137d23ee1df5485 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Wed, 24 Apr 2024 23:45:49 +0300 Subject: [PATCH 21/21] Auto-backup settings every 10 minutes. Increase backups limit to 50. --- src/endpoints/settings.js | 28 ++++++++++++++++++++++++++++ src/util.js | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/endpoints/settings.js b/src/endpoints/settings.js index c3d0caa74..a907fab5b 100644 --- a/src/endpoints/settings.js +++ b/src/endpoints/settings.js @@ -1,6 +1,7 @@ const fs = require('fs'); const path = require('path'); const express = require('express'); +const _ = require('lodash'); const writeFileAtomicSync = require('write-file-atomic').sync; const { PUBLIC_DIRECTORIES, SETTINGS_FILE } = require('../constants'); const { getConfigValue, generateTimestamp, removeOldBackups } = require('../util'); @@ -10,6 +11,32 @@ const { getAllUserHandles, getUserDirectories } = require('../users'); const ENABLE_EXTENSIONS = getConfigValue('enableExtensions', true); const ENABLE_ACCOUNTS = getConfigValue('enableUserAccounts', false); +// 10 minutes +const AUTOSAVE_INTERVAL = 10 * 60 * 1000; + +/** + * Map of functions to trigger settings autosave for a user. + * @type {Map} + */ +const AUTOSAVE_FUNCTIONS = new Map(); + +/** + * Triggers autosave for a user every 10 minutes. + * @param {string} handle User handle + * @returns {void} + */ +function triggerAutoSave(handle) { + if (!AUTOSAVE_FUNCTIONS.has(handle)) { + const throttledAutoSave = _.throttle(() => backupUserSettings(handle), AUTOSAVE_INTERVAL); + AUTOSAVE_FUNCTIONS.set(handle, throttledAutoSave); + } + + const functionToCall = AUTOSAVE_FUNCTIONS.get(handle); + if (functionToCall) { + functionToCall(); + } +} + /** * Reads and parses files from a directory. * @param {string} directoryPath Path to the directory @@ -121,6 +148,7 @@ router.post('/save', jsonParser, function (request, response) { try { const pathToSettings = path.join(request.user.directories.root, SETTINGS_FILE); writeFileAtomicSync(pathToSettings, JSON.stringify(request.body, null, 4), 'utf8'); + triggerAutoSave(request.user.profile.handle); response.send({ result: 'ok' }); } catch (err) { console.log(err); diff --git a/src/util.js b/src/util.js index ab19f3ccf..e1fd05e82 100644 --- a/src/util.js +++ b/src/util.js @@ -350,7 +350,7 @@ function generateTimestamp() { * @param {string} prefix */ function removeOldBackups(prefix) { - const MAX_BACKUPS = 25; + const MAX_BACKUPS = 50; let files = fs.readdirSync(PUBLIC_DIRECTORIES.backups).filter(f => f.startsWith(prefix)); if (files.length > MAX_BACKUPS) {