diff --git a/.github/workflows/update-docs.yml b/.github/workflows/update-docs.yml deleted file mode 100644 index 567cac607..000000000 --- a/.github/workflows/update-docs.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Update SillyTavern-Docs - -on: - push: - branches: - - main - -jobs: - update_docs: - runs-on: ubuntu-latest - - steps: - - name: Checkout current repository - uses: actions/checkout@v2 - - - name: Checkout SillyTavern-Docs repository - uses: actions/checkout@v2 - with: - repository: SillyTavern/SillyTavern-Docs - path: SillyTavern-Docs - - - name: Clone SillyTavern wiki into SillyTavern-Docs/extensions - run: rm -rf SillyTavern-Docs/extensions && git clone https://github.com/SillyTavern/SillyTavern.wiki.git SillyTavern-Docs/extensions && rm -rf SillyTavern-Docs/extensions/.git - - - name: Copy files - run: | - cp public/notes/content.md SillyTavern-Docs/guidebook.md - cp faq.md SillyTavern-Docs/faq.md - cp readme.md SillyTavern-Docs/readme.md - cp public/notes/update.md SillyTavern-Docs/update.md - - - name: Deploy to external repository - uses: cpina/github-action-push-to-another-repository@main - env: - SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }} - with: - # GitHub Action output files - source-directory: SillyTavern-Docs/ - destination-github-username: SillyTavern - destination-repository-name: SillyTavern-Docs - user-email: github-actions[bot]@users.noreply.github.com - user-name: "GitHub Actions" - target-branch: "main" diff --git a/Update-Instructions.txt b/Update-Instructions.txt index f153660b2..6e7071184 100644 --- a/Update-Instructions.txt +++ b/Update-Instructions.txt @@ -33,7 +33,14 @@ If you insist on installing via a zip, here is the tedious process for doing the 2. Unzip it into a folder OUTSIDE of your current ST installation. 3. Do the usual setup procedure for your OS to install the NodeJS requirements. -4. Copy the following files/folders as necessary(*) from your old ST installation: +4a. Updating 1.12.0 and above + +Copy the user data directory from your data root into the data root of the new install. + +By default: /data/default-user + +4a. Migrating from <1.12.0 to >=1.20.0 +Copy the following files/folders as necessary(*) from your old ST installation: - Assets - Backgrounds @@ -54,16 +61,15 @@ If you insist on installing via a zip, here is the tedious process for doing the - Worlds - User - settings.json - - secrets.json <---- this one is in the base folder, not /public/ + - secrets.json <---- This one is in the base folder, not /public/ (*) 'As necessary' = "If you made any custom content related to those folders". None of the folders are mandatory, so only copy what you need. **NB: DO NOT COPY THE ENTIRE /PUBLIC/ FOLDER.** Doing so could break the new install and prevent new features from being present. + Paste those items into the /data/default-user folder of the new install. -5. Paste those items into the /Public/ folder of the new install. +5. Start SillyTavern once again with the method appropriate to your OS, and pray you got it right. -6. Start SillyTavern once again with the method appropriate to your OS, and pray you got it right. - -7. If everything shows up, you can safely delete the old ST folder. +6. If everything shows up, you can safely delete the old ST folder. diff --git a/default/config.yaml b/default/config.yaml index 8966447d0..355573d96 100644 --- a/default/config.yaml +++ b/default/config.yaml @@ -9,6 +9,8 @@ port: 8000 # -- SECURITY CONFIGURATION -- # Toggle whitelist mode whitelistMode: true +# Whitelist will also verify IP in X-Forwarded-For / X-Real-IP headers +enableForwardedWhitelist: true # Whitelist of allowed IP addresses whitelist: - 127.0.0.1 diff --git a/public/css/mobile-styles.css b/public/css/mobile-styles.css index 68534d8a2..8456d54a0 100644 --- a/public/css/mobile-styles.css +++ b/public/css/mobile-styles.css @@ -231,9 +231,11 @@ backdrop-filter: blur(calc(var(--SmartThemeBlurStrength) * 2)); } + /* #right-nav-panel { padding-right: 15px; } + */ #floatingPrompt, #cfgConfig, diff --git a/public/css/toggle-dependent.css b/public/css/toggle-dependent.css index 30fb33bcf..834e31c31 100644 --- a/public/css/toggle-dependent.css +++ b/public/css/toggle-dependent.css @@ -433,14 +433,6 @@ body.expandMessageActions .mes .mes_buttons .extraMesButtonsHint { display: none !important; } -#openai_image_inlining:not(:checked)~#image_inlining_hint { - display: none; -} - -#openai_image_inlining:checked~#image_inlining_hint { - display: block; -} - #smooth_streaming:not(:checked)~#smooth_streaming_speed_control { display: none; } diff --git a/public/index.html b/public/index.html index 44c2254c4..b3642df50 100644 --- a/public/index.html +++ b/public/index.html @@ -4982,10 +4982,11 @@ - + + + +
@@ -6092,4 +6093,4 @@ - \ No newline at end of file + diff --git a/public/script.js b/public/script.js index d731d8b28..41f3a8b45 100644 --- a/public/script.js +++ b/public/script.js @@ -453,6 +453,7 @@ export const event_types = { CHARACTER_DUPLICATED: 'character_duplicated', SMOOTH_STREAM_TOKEN_RECEIVED: 'smooth_stream_token_received', FILE_ATTACHMENT_DELETED: 'file_attachment_deleted', + WORLDINFO_FORCE_ACTIVATE: 'worldinfo_force_activate', }; export const eventSource = new EventEmitter(); @@ -1844,7 +1845,7 @@ function messageFormatting(mes, ch_name, isSystem, isUser, messageId) { */ if (!power_user.allow_name2_display && ch_name && !isUser && !isSystem) { - mes = mes.replace(new RegExp(`(^|\n)${ch_name}:`, 'g'), '$1'); + mes = mes.replace(new RegExp(`(^|\n)${escapeRegex(ch_name)}:`, 'g'), '$1'); } /** @type {any} */ diff --git a/public/scripts/chats.js b/public/scripts/chats.js index a4082f6f0..0d2a9706a 100644 --- a/public/scripts/chats.js +++ b/public/scripts/chats.js @@ -53,11 +53,11 @@ import { ScraperManager } from './scrapers.js'; * @returns {Promise} Converted file text */ -const fileSizeLimit = 1024 * 1024 * 10; // 10 MB +const fileSizeLimit = 1024 * 1024 * 100; // 100 MB const ATTACHMENT_SOURCE = { GLOBAL: 'global', - CHAT: 'chat', CHARACTER: 'character', + CHAT: 'chat', }; /** @@ -592,9 +592,10 @@ async function deleteMessageImage() { /** * Deletes file from the server. * @param {string} url Path to the file on the server + * @param {boolean} [silent=false] If true, do not show error messages * @returns {Promise} True if file was deleted, false otherwise. */ -async function deleteFileFromServer(url) { +async function deleteFileFromServer(url, silent = false) { try { const result = await fetch('/api/files/delete', { method: 'POST', @@ -602,7 +603,7 @@ async function deleteFileFromServer(url) { body: JSON.stringify({ path: url }), }); - if (!result.ok) { + if (!result.ok && !silent) { const error = await result.text(); throw new Error(error); } @@ -669,6 +670,55 @@ async function editAttachment(attachment, source, callback) { callback(); } +/** + * Downloads an attachment to the user's device. + * @param {FileAttachment} attachment Attachment to download + */ +async function downloadAttachment(attachment) { + const fileText = attachment.text || (await getFileAttachment(attachment.url)); + const blob = new Blob([fileText], { type: 'text/plain' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = attachment.name; + a.click(); + URL.revokeObjectURL(url); +} + +/** + * Moves a file attachment to a different source. + * @param {FileAttachment} attachment Attachment to moves + * @param {string} source Source of the attachment + * @param {function} callback Success callback + * @returns {Promise} A promise that resolves when the attachment is moved. + */ +async function moveAttachment(attachment, source, callback) { + let selectedTarget = source; + const targets = getAvailableTargets(); + const template = $(await renderExtensionTemplateAsync('attachments', 'move-attachment', { name: attachment.name, targets })); + template.find('.moveAttachmentTarget').val(source).on('input', function () { + selectedTarget = String($(this).val()); + }); + + const result = await callGenericPopup(template, POPUP_TYPE.CONFIRM, '', { wide: false, large: false, okButton: 'Move', cancelButton: 'Cancel' }); + + if (result !== POPUP_RESULT.AFFIRMATIVE) { + console.debug('Move attachment cancelled'); + return; + } + + if (selectedTarget === source) { + console.debug('Move attachment cancelled: same source and target'); + return; + } + + const content = await getFileAttachment(attachment.url); + const file = new File([content], attachment.name, { type: 'text/plain' }); + await deleteAttachment(attachment, source, () => { }, false); + await uploadFileAttachmentToServer(file, selectedTarget); + callback(); +} + /** * Deletes an attachment from the server and the chat. * @param {FileAttachment} attachment Attachment to delete @@ -702,7 +752,8 @@ async function deleteAttachment(attachment, source, callback, confirm = true) { break; } - await deleteFileFromServer(attachment.url); + const silent = confirm === false; + await deleteFileFromServer(attachment.url, silent); callback(); } @@ -756,12 +807,15 @@ async function openAttachmentManager() { for (const attachment of sortedAttachmentList) { const attachmentTemplate = template.find('.attachmentListItemTemplate .attachmentListItem').clone(); + attachmentTemplate.find('.attachmentFileIcon').attr('title', attachment.url); attachmentTemplate.find('.attachmentListItemName').text(attachment.name); attachmentTemplate.find('.attachmentListItemSize').text(humanFileSize(attachment.size)); attachmentTemplate.find('.attachmentListItemCreated').text(new Date(attachment.created).toLocaleString()); attachmentTemplate.find('.viewAttachmentButton').on('click', () => openFilePopup(attachment)); attachmentTemplate.find('.editAttachmentButton').on('click', () => editAttachment(attachment, source, renderAttachments)); attachmentTemplate.find('.deleteAttachmentButton').on('click', () => deleteAttachment(attachment, source, renderAttachments)); + attachmentTemplate.find('.downloadAttachmentButton').on('click', () => downloadAttachment(attachment)); + attachmentTemplate.find('.moveAttachmentButton').on('click', () => moveAttachment(attachment, source, renderAttachments)); template.find(sources[source]).append(attachmentTemplate); } } @@ -860,6 +914,50 @@ async function openAttachmentManager() { template.find('.chatAttachmentsName').text(chatName); } + function addDragAndDrop() { + $(document.body).on('dragover', '.dialogue_popup', (event) => { + event.preventDefault(); + event.stopPropagation(); + $(event.target).closest('.dialogue_popup').addClass('dragover'); + }); + + $(document.body).on('dragleave', '.dialogue_popup', (event) => { + event.preventDefault(); + event.stopPropagation(); + $(event.target).closest('.dialogue_popup').removeClass('dragover'); + }); + + $(document.body).on('drop', '.dialogue_popup', async (event) => { + event.preventDefault(); + event.stopPropagation(); + $(event.target).closest('.dialogue_popup').removeClass('dragover'); + + const files = Array.from(event.originalEvent.dataTransfer.files); + let selectedTarget = ATTACHMENT_SOURCE.GLOBAL; + const targets = getAvailableTargets(); + + const targetSelectTemplate = $(await renderExtensionTemplateAsync('attachments', 'files-dropped', { count: files.length, targets: targets })); + targetSelectTemplate.find('.droppedFilesTarget').on('input', function () { + selectedTarget = String($(this).val()); + }); + const result = await callGenericPopup(targetSelectTemplate, POPUP_TYPE.CONFIRM, '', { wide: false, large: false, okButton: 'Upload', cancelButton: 'Cancel' }); + if (result !== POPUP_RESULT.AFFIRMATIVE) { + console.log('File upload cancelled'); + return; + } + for (const file of files) { + await uploadFileAttachmentToServer(file, selectedTarget); + } + renderAttachments(); + }); + } + + function removeDragAndDrop() { + $(document.body).off('dragover', '.shadow_popup'); + $(document.body).off('dragleave', '.shadow_popup'); + $(document.body).off('drop', '.shadow_popup'); + } + let sortField = localStorage.getItem('DataBank_sortField') || 'created'; let sortOrder = localStorage.getItem('DataBank_sortOrder') || 'desc'; let filterString = ''; @@ -883,10 +981,34 @@ async function openAttachmentManager() { }); const cleanupFn = await renderButtons(); + await verifyAttachments(); await renderAttachments(); + addDragAndDrop(); await callGenericPopup(template, POPUP_TYPE.TEXT, '', { wide: true, large: true, okButton: 'Close' }); cleanupFn(); + removeDragAndDrop(); +} + +/** + * Gets a list of available targets for attachments. + * @returns {string[]} List of available targets + */ +function getAvailableTargets() { + const targets = Object.values(ATTACHMENT_SOURCE); + + const isNotCharacter = this_chid === undefined || selected_group; + const isNotInChat = getCurrentChatId() === undefined; + + if (isNotCharacter) { + targets.splice(targets.indexOf(ATTACHMENT_SOURCE.CHARACTER), 1); + } + + if (isNotInChat) { + targets.splice(targets.indexOf(ATTACHMENT_SOURCE.CHAT), 1); + } + + return targets; } /** @@ -1039,6 +1161,48 @@ export function getDataBankAttachmentsForSource(source) { } } +/** + * Verifies all attachments in the Data Bank. + * @returns {Promise} A promise that resolves when attachments are verified. + */ +async function verifyAttachments() { + for (const source of Object.values(ATTACHMENT_SOURCE)) { + await verifyAttachmentsForSource(source); + } +} + +/** + * Verifies all attachments for a specific source. + * @param {string} source Attachment source + * @returns {Promise} A promise that resolves when attachments are verified. + */ +async function verifyAttachmentsForSource(source) { + try { + const attachments = getDataBankAttachmentsForSource(source); + const urls = attachments.map(a => a.url); + const response = await fetch('/api/files/verify', { + method: 'POST', + headers: getRequestHeaders(), + body: JSON.stringify({ urls }), + }); + + if (!response.ok) { + const error = await response.text(); + throw new Error(error); + } + + const verifiedUrls = await response.json(); + for (const attachment of attachments) { + if (verifiedUrls[attachment.url] === false) { + console.log('Deleting orphaned attachment', attachment); + await deleteAttachment(attachment, source, () => { }, false); + } + } + } catch (error) { + console.error('Attachment verification failed', error); + } +} + /** * Registers a file converter function. * @param {string} mimeType MIME type diff --git a/public/scripts/extensions/attachments/files-dropped.html b/public/scripts/extensions/attachments/files-dropped.html new file mode 100644 index 000000000..9fd014ded --- /dev/null +++ b/public/scripts/extensions/attachments/files-dropped.html @@ -0,0 +1,8 @@ +
+ Save {{count}} file(s) to... + +
diff --git a/public/scripts/extensions/attachments/index.js b/public/scripts/extensions/attachments/index.js index a7f58dbc7..ab3db9419 100644 --- a/public/scripts/extensions/attachments/index.js +++ b/public/scripts/extensions/attachments/index.js @@ -1,6 +1,9 @@ import { renderExtensionTemplateAsync } from '../../extensions.js'; +import { registerSlashCommand } from '../../slash-commands.js'; jQuery(async () => { const buttons = await renderExtensionTemplateAsync('attachments', 'buttons', {}); $('#extensionsMenu').prepend(buttons); + + registerSlashCommand('db', () => document.getElementById('manageAttachments')?.click(), ['databank', 'data-bank'], '– open the data bank', true, true); }); diff --git a/public/scripts/extensions/attachments/manager.html b/public/scripts/extensions/attachments/manager.html index ecfbe2fbb..68ad732a2 100644 --- a/public/scripts/extensions/attachments/manager.html +++ b/public/scripts/extensions/attachments/manager.html @@ -7,8 +7,13 @@
These files will be available for extensions that support attachments (e.g. Vector Storage).
-
- Supported file types: Plain Text, PDF, Markdown, HTML, EPUB. +
+ + Supported file types: Plain Text, PDF, Markdown, HTML, EPUB. + + + Drag and drop files here to upload. +
@@ -102,7 +107,9 @@
+
+
diff --git a/public/scripts/extensions/attachments/manifest.json b/public/scripts/extensions/attachments/manifest.json index 2037168c2..27f55f77c 100644 --- a/public/scripts/extensions/attachments/manifest.json +++ b/public/scripts/extensions/attachments/manifest.json @@ -1,5 +1,5 @@ { - "display_name": "Chat Attachments", + "display_name": "Data Bank (Chat Attachments)", "loading_order": 3, "requires": [], "optional": [], diff --git a/public/scripts/extensions/attachments/move-attachment.html b/public/scripts/extensions/attachments/move-attachment.html new file mode 100644 index 000000000..56e265269 --- /dev/null +++ b/public/scripts/extensions/attachments/move-attachment.html @@ -0,0 +1,8 @@ +
+ Move {{name}} to... + +
diff --git a/public/scripts/extensions/expressions/settings.html b/public/scripts/extensions/expressions/settings.html index 4a7347a74..e8b1484b2 100644 --- a/public/scripts/extensions/expressions/settings.html +++ b/public/scripts/extensions/expressions/settings.html @@ -78,7 +78,7 @@ Remove all image overrides
-

Hint: Create new folder in the public/characters/ folder and name it as the name of the character. +

Hint: Create new folder in the /characters/ folder of your user data directory and name it as the name of the character. Put images with expressions there. File names should follow the pattern: [expression_label].[image_format]

diff --git a/public/scripts/extensions/quick-reply/index.js b/public/scripts/extensions/quick-reply/index.js index 28053ec64..2f1bcf908 100644 --- a/public/scripts/extensions/quick-reply/index.js +++ b/public/scripts/extensions/quick-reply/index.js @@ -1,4 +1,4 @@ -import { chat_metadata, eventSource, event_types, getRequestHeaders } from '../../../script.js'; +import { chat, chat_metadata, eventSource, event_types, getRequestHeaders } from '../../../script.js'; import { extension_settings } from '../../extensions.js'; import { QuickReplyApi } from './api/QuickReplyApi.js'; import { AutoExecuteHandler } from './src/AutoExecuteHandler.js'; @@ -240,7 +240,12 @@ const onUserMessage = async () => { }; eventSource.on(event_types.USER_MESSAGE_RENDERED, (...args)=>executeIfReadyElseQueue(onUserMessage, args)); -const onAiMessage = async () => { +const onAiMessage = async (messageId) => { + if (['...'].includes(chat[messageId]?.mes)) { + log('QR auto-execution suppressed for swiped message'); + return; + } + await autoExec.handleAi(); }; eventSource.on(event_types.CHARACTER_MESSAGE_RENDERED, (...args)=>executeIfReadyElseQueue(onAiMessage, args)); diff --git a/public/scripts/extensions/stable-diffusion/index.js b/public/scripts/extensions/stable-diffusion/index.js index d7b0cfc0b..88fdbad40 100644 --- a/public/scripts/extensions/stable-diffusion/index.js +++ b/public/scripts/extensions/stable-diffusion/index.js @@ -1657,6 +1657,10 @@ async function loadNovelModels() { value: 'safe-diffusion', text: 'NAI Diffusion Anime V1 (Curated)', }, + { + value: 'nai-diffusion-furry-3', + text: 'NAI Diffusion Furry V3', + }, { value: 'nai-diffusion-furry', text: 'NAI Diffusion Furry', diff --git a/public/scripts/extensions/vectors/index.js b/public/scripts/extensions/vectors/index.js index 6ff5c6af5..ca2ddbdeb 100644 --- a/public/scripts/extensions/vectors/index.js +++ b/public/scripts/extensions/vectors/index.js @@ -23,6 +23,7 @@ import { collapseNewlines } from '../../power-user.js'; import { SECRET_KEYS, secret_state, writeSecret } from '../../secrets.js'; import { getDataBankAttachments, getFileAttachment } from '../../chats.js'; import { debounce, getStringHash as calculateHash, waitUntilCondition, onlyUnique, splitRecursive } from '../../utils.js'; +import { getSortedEntries } from '../../world-info.js'; const MODULE_NAME = 'vectors'; @@ -66,6 +67,11 @@ const settings = { file_position_db: extension_prompt_types.IN_PROMPT, file_depth_db: 4, file_depth_role_db: extension_prompt_roles.SYSTEM, + + // For World Info + enabled_world_info: false, + enabled_for_all: false, + max_entries: 5, }; const moduleWorker = new ModuleWorkerWrapper(synchronizeChat); @@ -281,8 +287,10 @@ async function synchronizeChat(batchSize = 5) { } } -// Cache object for storing hash values -const hashCache = {}; +/** + * @type {Map} Cache object for storing hash values + */ +const hashCache = new Map(); /** * Gets the hash value for a given string @@ -291,15 +299,15 @@ const hashCache = {}; */ function getStringHash(str) { // Check if the hash is already in the cache - if (Object.hasOwn(hashCache, str)) { - return hashCache[str]; + if (hashCache.has(str)) { + return hashCache.get(str); } // Calculate the hash value const hash = calculateHash(str); // Store the hash in the cache - hashCache[str] = hash; + hashCache.set(str, hash); return hash; } @@ -472,6 +480,10 @@ async function rearrangeChat(chat) { await processFiles(chat); } + if (settings.enabled_world_info) { + await activateWorldInfo(chat); + } + if (!settings.enabled_chats) { return; } @@ -845,6 +857,7 @@ async function purgeVectorIndex(collectionId) { function toggleSettings() { $('#vectors_files_settings').toggle(!!settings.enabled_files); $('#vectors_chats_settings').toggle(!!settings.enabled_chats); + $('#vectors_world_info_settings').toggle(!!settings.enabled_world_info); $('#together_vectorsModel').toggle(settings.source === 'togetherai'); $('#openai_vectorsModel').toggle(settings.source === 'openai'); $('#cohere_vectorsModel').toggle(settings.source === 'cohere'); @@ -934,6 +947,111 @@ async function onPurgeFilesClick() { } } +async function activateWorldInfo(chat) { + if (!settings.enabled_world_info) { + console.debug('Vectors: Disabled for World Info'); + return; + } + + const entries = await getSortedEntries(); + + if (!Array.isArray(entries) || entries.length === 0) { + console.debug('Vectors: No WI entries found'); + return; + } + + // Group entries by "world" field + const groupedEntries = {}; + + for (const entry of entries) { + // Skip orphaned entries. Is it even possible? + if (!entry.world) { + console.debug('Vectors: Skipped orphaned WI entry', entry); + continue; + } + + // Skip disabled entries + if (entry.disable) { + console.debug('Vectors: Skipped disabled WI entry', entry); + continue; + } + + // Skip entries without content + if (!entry.content) { + console.debug('Vectors: Skipped WI entry without content', entry); + continue; + } + + // Skip non-vectorized entries + if (!entry.vectorized && !settings.enabled_for_all) { + console.debug('Vectors: Skipped non-vectorized WI entry', entry); + continue; + } + + if (!Object.hasOwn(groupedEntries, entry.world)) { + groupedEntries[entry.world] = []; + } + + groupedEntries[entry.world].push(entry); + } + + const collectionIds = []; + + if (Object.keys(groupedEntries).length === 0) { + console.debug('Vectors: No WI entries to synchronize'); + return; + } + + // Synchronize collections + for (const world in groupedEntries) { + const collectionId = `world_${getStringHash(world)}`; + const hashesInCollection = await getSavedHashes(collectionId); + const newEntries = groupedEntries[world].filter(x => !hashesInCollection.includes(getStringHash(x.content))); + const deletedHashes = hashesInCollection.filter(x => !groupedEntries[world].some(y => getStringHash(y.content) === x)); + + if (newEntries.length > 0) { + console.log(`Vectors: Found ${newEntries.length} new WI entries for world ${world}`); + await insertVectorItems(collectionId, newEntries.map(x => ({ hash: getStringHash(x.content), text: x.content, index: x.uid }))); + } + + if (deletedHashes.length > 0) { + console.log(`Vectors: Deleted ${deletedHashes.length} old hashes for world ${world}`); + await deleteVectorItems(collectionId, deletedHashes); + } + + collectionIds.push(collectionId); + } + + // Perform a multi-query + const queryText = await getQueryText(chat); + + if (queryText.length === 0) { + console.debug('Vectors: No text to query for WI'); + return; + } + + const queryResults = await queryMultipleCollections(collectionIds, queryText, settings.max_entries); + const activatedHashes = Object.values(queryResults).flatMap(x => x.hashes).filter(onlyUnique); + const activatedEntries = []; + + // Activate entries found in the query results + for (const entry of entries) { + const hash = getStringHash(entry.content); + + if (activatedHashes.includes(hash)) { + activatedEntries.push(entry); + } + } + + if (activatedEntries.length === 0) { + console.debug('Vectors: No activated WI entries found'); + return; + } + + console.log(`Vectors: Activated ${activatedEntries.length} WI entries`, activatedEntries); + await eventSource.emit(event_types.WORLDINFO_FORCE_ACTIVATE, activatedEntries); +} + jQuery(async () => { if (!extension_settings.vectors) { extension_settings.vectors = settings; @@ -1134,6 +1252,25 @@ jQuery(async () => { saveSettingsDebounced(); }); + $('#vectors_enabled_world_info').prop('checked', settings.enabled_world_info).on('input', () => { + settings.enabled_world_info = !!$('#vectors_enabled_world_info').prop('checked'); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + toggleSettings(); + }); + + $('#vectors_enabled_for_all').prop('checked', settings.enabled_for_all).on('input', () => { + settings.enabled_for_all = !!$('#vectors_enabled_for_all').prop('checked'); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + }); + + $('#vectors_max_entries').val(settings.max_entries).on('input', () => { + settings.max_entries = Number($('#vectors_max_entries').val()); + Object.assign(extension_settings.vectors, settings); + saveSettingsDebounced(); + }); + const validSecret = !!secret_state[SECRET_KEYS.NOMICAI]; const placeholder = validSecret ? '✔️ Key saved' : '❌ Missing key'; $('#api_key_nomicai').attr('placeholder', placeholder); diff --git a/public/scripts/extensions/vectors/settings.html b/public/scripts/extensions/vectors/settings.html index 02499d120..bcaf1c06e 100644 --- a/public/scripts/extensions/vectors/settings.html +++ b/public/scripts/extensions/vectors/settings.html @@ -97,6 +97,46 @@
+

+ World Info settings +

+ + + +
+
+ +
    +
  • + Checked: all entries except ❌ status can be activated. +
    • +
  • + Unchecked: only entries with 🔗 status can be activated. +
    • +
+
+
+
+ +
+
+ + +
+
+ +
+
+
+

File vectorization settings

diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js index 2932e9050..767f923d0 100644 --- a/public/scripts/world-info.js +++ b/public/scripts/world-info.js @@ -82,6 +82,11 @@ class WorldInfoBuffer { /** @typedef {{scanDepth?: number, caseSensitive?: boolean, matchWholeWords?: boolean}} WIScanEntry The entry that triggered the scan */ // End typedef area + /** + * @type {object[]} Array of entries that need to be activated no matter what + */ + static externalActivations = []; + /** * @type {string[]} Array of messages sorted by ascending depth */ @@ -220,6 +225,23 @@ class WorldInfoBuffer { getDepth() { return world_info_depth + this.#skew; } + + /** + * Check if the current entry is externally activated. + * @param {object} entry WI entry to check + * @returns {boolean} True if the entry is forcefully activated + */ + isExternallyActivated(entry) { + // Entries could be copied with structuredClone, so we need to compare them by string representation + return WorldInfoBuffer.externalActivations.some(x => JSON.stringify(x) === JSON.stringify(entry)); + } + + /** + * Clears the force activations buffer. + */ + cleanExternalActivations() { + WorldInfoBuffer.externalActivations.splice(0, WorldInfoBuffer.externalActivations.length); + } } export function getWorldInfoSettings() { @@ -362,6 +384,10 @@ function setWorldInfoSettings(settings, data) { $('.chat_lorebook_button').toggleClass('world_set', hasWorldInfo); }); + eventSource.on(event_types.WORLDINFO_FORCE_ACTIVATE, (entries) => { + WorldInfoBuffer.externalActivations.push(...entries); + }); + // Add slash commands registerWorldInfoSlashCommands(); } @@ -564,6 +590,7 @@ function registerWorldInfoSlashCommands() { return ''; } + registerSlashCommand('world', onWorldInfoChange, [], '[optional state=off|toggle] [optional silent=true] (optional name) – sets active World, or unsets if no args provided, use state=off and state=toggle to deactivate or toggle a World, use silent=true to suppress toast messages', true, true); registerSlashCommand('getchatbook', getChatBookCallback, ['getchatlore', 'getchatwi'], '– get a name of the chat-bound lorebook or create a new one if was unbound, and pass it down the pipe', true, true); registerSlashCommand('findentry', findBookEntryCallback, ['findlore', 'findwi'], '(file=bookName field=field [texts]) – find a UID of the record from the specified book using the fuzzy match of a field value (default: key) and pass it down the pipe, e.g. /findentry file=chatLore field=key Shadowfang', true, true); registerSlashCommand('getentryfield', getEntryFieldCallback, ['getlorefield', 'getwifield'], '(file=bookName field=field [UID]) – get a field value (default: content) of the record with the UID from the specified book and pass it down the pipe, e.g. /getentryfield file=chatLore field=content 123', true, true); @@ -964,6 +991,7 @@ const originalDataKeyMap = { 'caseSensitive': 'extensions.case_sensitive', 'scanDepth': 'extensions.scan_depth', 'automationId': 'extensions.automation_id', + 'vectorized': 'extensions.vectorized', }; function setOriginalDataValue(data, uid, key, value) { @@ -1071,6 +1099,16 @@ function getWorldEntry(name, data, entry) { ); } + // Verify names to exist in the system + if (data.entries[uid]?.characterFilter?.names?.length > 0) { + for (const name of [...data.entries[uid].characterFilter.names]) { + if (!getContext().characters.find(x => x.avatar.replace(/\.[^/.]+$/, '') === name)) { + console.warn(`World Info: Character ${name} not found. Removing from the entry filter.`, entry); + data.entries[uid].characterFilter.names = data.entries[uid].characterFilter.names.filter(x => x !== name); + } + } + } + setOriginalDataValue(data, uid, 'character_filter', data.entries[uid].characterFilter); saveWorldInfo(name, data); }); @@ -1454,22 +1492,37 @@ function getWorldEntry(name, data, entry) { case 'constant': data.entries[uid].constant = true; data.entries[uid].disable = false; + data.entries[uid].vectorized = false; setOriginalDataValue(data, uid, 'enabled', true); setOriginalDataValue(data, uid, 'constant', true); + setOriginalDataValue(data, uid, 'extensions.vectorized', false); template.removeClass('disabledWIEntry'); break; case 'normal': data.entries[uid].constant = false; data.entries[uid].disable = false; + data.entries[uid].vectorized = false; setOriginalDataValue(data, uid, 'enabled', true); setOriginalDataValue(data, uid, 'constant', false); + setOriginalDataValue(data, uid, 'extensions.vectorized', false); + template.removeClass('disabledWIEntry'); + break; + case 'vectorized': + data.entries[uid].constant = false; + data.entries[uid].disable = false; + data.entries[uid].vectorized = true; + setOriginalDataValue(data, uid, 'enabled', true); + setOriginalDataValue(data, uid, 'constant', false); + setOriginalDataValue(data, uid, 'extensions.vectorized', true); template.removeClass('disabledWIEntry'); break; case 'disabled': data.entries[uid].constant = false; data.entries[uid].disable = true; + data.entries[uid].vectorized = false; setOriginalDataValue(data, uid, 'enabled', false); setOriginalDataValue(data, uid, 'constant', false); + setOriginalDataValue(data, uid, 'extensions.vectorized', false); template.addClass('disabledWIEntry'); break; } @@ -1480,6 +1533,8 @@ function getWorldEntry(name, data, entry) { const entryState = function () { if (entry.constant === true) { return 'constant'; + } else if (entry.vectorized === true) { + return 'vectorized'; } else if (entry.disable === true) { return 'disabled'; } else { @@ -1719,6 +1774,7 @@ const newEntryTemplate = { comment: '', content: '', constant: false, + vectorized: false, selective: true, selectiveLogic: world_info_logic.AND_ANY, addMemo: false, @@ -1925,7 +1981,7 @@ async function getCharacterLore() { } const data = await loadWorldInfoData(worldName); - const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]) : []; + const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x => ({ ...x, world: worldName })) : []; entries = entries.concat(newEntries); } @@ -1941,7 +1997,7 @@ async function getGlobalLore() { let entries = []; for (const worldName of selected_world_info) { const data = await loadWorldInfoData(worldName); - const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]) : []; + const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x => ({ ...x, world: worldName })) : []; entries = entries.concat(newEntries); } @@ -1963,14 +2019,14 @@ async function getChatLore() { } const data = await loadWorldInfoData(chatWorld); - const entries = data ? Object.keys(data.entries).map((x) => data.entries[x]) : []; + const entries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x => ({ ...x, world: chatWorld })) : []; console.debug(`Chat lore has ${entries.length} entries`); return entries; } -async function getSortedEntries() { +export async function getSortedEntries() { try { const globalLore = await getGlobalLore(); const characterLore = await getCharacterLore(); @@ -2098,7 +2154,7 @@ async function checkWorldInfo(chat, maxContext) { continue; } - if (entry.constant) { + if (entry.constant || buffer.isExternallyActivated(entry)) { entry.content = substituteParams(entry.content); activatedNow.add(entry); continue; @@ -2295,6 +2351,8 @@ async function checkWorldInfo(chat, maxContext) { context.setExtensionPrompt(NOTE_MODULE_NAME, ANWithWI, chat_metadata[metadata_keys.position], chat_metadata[metadata_keys.depth], extension_settings.note.allowWIScan, chat_metadata[metadata_keys.role]); } + buffer.cleanExternalActivations(); + return { worldInfoBefore, worldInfoAfter, WIDepthEntries, allActivatedEntries }; } @@ -2381,6 +2439,7 @@ function convertAgnaiMemoryBook(inputObj) { content: entry.entry, constant: false, selective: false, + vectorized: false, selectiveLogic: world_info_logic.AND_ANY, order: entry.weight, position: 0, @@ -2415,6 +2474,7 @@ function convertRisuLorebook(inputObj) { content: entry.content, constant: entry.alwaysActive, selective: entry.selective, + vectorized: false, selectiveLogic: world_info_logic.AND_ANY, order: entry.insertorder, position: world_info_position.before, @@ -2454,6 +2514,7 @@ function convertNovelLorebook(inputObj) { content: entry.text, constant: false, selective: false, + vectorized: false, selectiveLogic: world_info_logic.AND_ANY, order: entry.contextConfig?.budgetPriority ?? 0, position: 0, @@ -2510,6 +2571,7 @@ function convertCharacterBook(characterBook) { matchWholeWords: entry.extensions?.match_whole_words ?? null, automationId: entry.extensions?.automation_id ?? '', role: entry.extensions?.role ?? extension_prompt_roles.SYSTEM, + vectorized: entry.extensions?.vectorized ?? false, }; }); @@ -2785,11 +2847,6 @@ function assignLorebookToChat() { jQuery(() => { - $(document).ready(function () { - registerSlashCommand('world', onWorldInfoChange, [], '[optional state=off|toggle] [optional silent=true] (optional name) – sets active World, or unsets if no args provided, use state=off and state=toggle to deactivate or toggle a World, use silent=true to suppress toast messages', true, true); - }); - - $('#world_info').on('mousedown change', async function (e) { // If there's no world names, don't do anything if (world_names.length === 0) { diff --git a/public/style.css b/public/style.css index 6d0eabb82..51245f576 100644 --- a/public/style.css +++ b/public/style.css @@ -2780,6 +2780,11 @@ grammarly-extension { } } +.dialogue_popup.dragover { + filter: brightness(1.1) saturate(1.1); + outline: 3px dashed var(--SmartThemeBorderColor); +} + #bgtest { display: none; width: 100vw; diff --git a/server.js b/server.js index c75304f36..cbb74d900 100644 --- a/server.js +++ b/server.js @@ -263,10 +263,14 @@ app.get('/login', async (request, response) => { return response.redirect('/'); } - const autoLogin = await userModule.tryAutoLogin(request); + try { + const autoLogin = await userModule.tryAutoLogin(request); - if (autoLogin) { - return response.redirect('/'); + if (autoLogin) { + return response.redirect('/'); + } + } catch (error) { + console.error('Error during auto-login:', error); } return response.sendFile('login.html', { root: path.join(process.cwd(), 'public') }); diff --git a/src/endpoints/assets.js b/src/endpoints/assets.js index 78f5270a7..09cc1aa71 100644 --- a/src/endpoints/assets.js +++ b/src/endpoints/assets.js @@ -56,6 +56,8 @@ function validateAssetFileName(inputFilename) { * @returns {string[]} - The array of files */ function getFiles(dir, files = []) { + if (!fs.existsSync(dir)) return files; + // Get an array of all files and directories in the passed directory using fs.readdirSync const fileList = fs.readdirSync(dir, { withFileTypes: true }); // Create the full path of the file/directory by concatenating the passed directory and file/directory name diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js index 22806cbf0..0e9598827 100644 --- a/src/endpoints/backends/text-completions.js +++ b/src/endpoints/backends/text-completions.js @@ -325,7 +325,7 @@ router.post('/generate', jsonParser, async function (request, response) { // Map InfermaticAI response to OAI completions format if (apiType === TEXTGEN_TYPES.INFERMATICAI) { - data['choices'] = (data?.choices || []).map(choice => ({ text: choice.message.content })); + data['choices'] = (data?.choices || []).map(choice => ({ text: choice?.message?.content || choice.text })); } return response.send(data); diff --git a/src/endpoints/characters.js b/src/endpoints/characters.js index 556a8fecc..929818ade 100644 --- a/src/endpoints/characters.js +++ b/src/endpoints/characters.js @@ -439,6 +439,7 @@ function convertWorldInfoToCharacterBook(name, entries) { case_sensitive: entry.caseSensitive ?? null, automation_id: entry.automationId ?? '', role: entry.role ?? 0, + vectorized: entry.vectorized ?? false, }, }; @@ -1097,7 +1098,7 @@ router.post('/export', jsonParser, async function (request, response) { const fileContent = await fsPromises.readFile(filename); const contentType = mime.lookup(filename) || 'image/png'; response.setHeader('Content-Type', contentType); - response.setHeader('Content-Disposition', `attachment; filename=${path.basename(filename)}`); + response.setHeader('Content-Disposition', `attachment; filename="${encodeURI(path.basename(filename))}"`); return response.send(fileContent); } case 'json': { diff --git a/src/endpoints/classify.js b/src/endpoints/classify.js index 5a9772e1d..758b95247 100644 --- a/src/endpoints/classify.js +++ b/src/endpoints/classify.js @@ -5,7 +5,10 @@ const TASK = 'text-classification'; const router = express.Router(); -const cacheObject = {}; +/** + * @type {Map} Cache for classification results + */ +const cacheObject = new Map(); router.post('/labels', jsonParser, async (req, res) => { try { @@ -23,15 +26,20 @@ router.post('/', jsonParser, async (req, res) => { try { const { text } = req.body; + /** + * Get classification result for a given text + * @param {string} text Text to classify + * @returns {Promise} Classification result + */ async function getResult(text) { - if (Object.hasOwn(cacheObject, text)) { - return cacheObject[text]; + if (cacheObject.has(text)) { + return cacheObject.get(text); } else { const module = await import('../transformers.mjs'); const pipe = await module.default.getPipeline(TASK); const result = await pipe(text, { topk: 5 }); result.sort((a, b) => b.score - a.score); - cacheObject[text] = result; + cacheObject.set(text, result); return result; } } diff --git a/src/endpoints/files.js b/src/endpoints/files.js index 1c66273bd..371381c21 100644 --- a/src/endpoints/files.js +++ b/src/endpoints/files.js @@ -57,4 +57,29 @@ router.post('/delete', jsonParser, async (request, response) => { } }); +router.post('/verify', jsonParser, async (request, response) => { + try { + if (!Array.isArray(request.body.urls)) { + return response.status(400).send('No URLs specified'); + } + + const verified = {}; + + for (const url of request.body.urls) { + const pathToVerify = path.join(request.user.directories.root, url); + if (!pathToVerify.startsWith(request.user.directories.files)) { + console.debug(`File verification: Invalid path: ${pathToVerify}`); + continue; + } + const fileExists = fs.existsSync(pathToVerify); + verified[url] = fileExists; + } + + return response.send(verified); + } catch (error) { + console.log(error); + return response.sendStatus(500); + } +}); + module.exports = { router }; diff --git a/src/endpoints/settings.js b/src/endpoints/settings.js index c3d0caa74..a907fab5b 100644 --- a/src/endpoints/settings.js +++ b/src/endpoints/settings.js @@ -1,6 +1,7 @@ const fs = require('fs'); const path = require('path'); const express = require('express'); +const _ = require('lodash'); const writeFileAtomicSync = require('write-file-atomic').sync; const { PUBLIC_DIRECTORIES, SETTINGS_FILE } = require('../constants'); const { getConfigValue, generateTimestamp, removeOldBackups } = require('../util'); @@ -10,6 +11,32 @@ const { getAllUserHandles, getUserDirectories } = require('../users'); const ENABLE_EXTENSIONS = getConfigValue('enableExtensions', true); const ENABLE_ACCOUNTS = getConfigValue('enableUserAccounts', false); +// 10 minutes +const AUTOSAVE_INTERVAL = 10 * 60 * 1000; + +/** + * Map of functions to trigger settings autosave for a user. + * @type {Map} + */ +const AUTOSAVE_FUNCTIONS = new Map(); + +/** + * Triggers autosave for a user every 10 minutes. + * @param {string} handle User handle + * @returns {void} + */ +function triggerAutoSave(handle) { + if (!AUTOSAVE_FUNCTIONS.has(handle)) { + const throttledAutoSave = _.throttle(() => backupUserSettings(handle), AUTOSAVE_INTERVAL); + AUTOSAVE_FUNCTIONS.set(handle, throttledAutoSave); + } + + const functionToCall = AUTOSAVE_FUNCTIONS.get(handle); + if (functionToCall) { + functionToCall(); + } +} + /** * Reads and parses files from a directory. * @param {string} directoryPath Path to the directory @@ -121,6 +148,7 @@ router.post('/save', jsonParser, function (request, response) { try { const pathToSettings = path.join(request.user.directories.root, SETTINGS_FILE); writeFileAtomicSync(pathToSettings, JSON.stringify(request.body, null, 4), 'utf8'); + triggerAutoSave(request.user.profile.handle); response.send({ result: 'ok' }); } catch (err) { console.log(err); diff --git a/src/middleware/whitelist.js b/src/middleware/whitelist.js index def408650..24c1af8e5 100644 --- a/src/middleware/whitelist.js +++ b/src/middleware/whitelist.js @@ -6,6 +6,7 @@ const { getIpFromRequest } = require('../express-common'); const { color, getConfigValue } = require('../util'); const whitelistPath = path.join(process.cwd(), './whitelist.txt'); +const enableForwardedWhitelist = getConfigValue('enableForwardedWhitelist', false); let whitelist = getConfigValue('whitelist', []); let knownIPs = new Set(); @@ -24,14 +25,18 @@ if (fs.existsSync(whitelistPath)) { * @returns {string|undefined} The client IP address */ function getForwardedIp(req) { + if (!enableForwardedWhitelist) { + return undefined; + } + // Check if X-Real-IP is available if (req.headers['x-real-ip']) { - return req.headers['x-real-ip']; + return req.headers['x-real-ip'].toString(); } // Check for X-Forwarded-For and parse if available if (req.headers['x-forwarded-for']) { - const ipList = req.headers['x-forwarded-for'].split(',').map(ip => ip.trim()); + const ipList = req.headers['x-forwarded-for'].toString().split(',').map(ip => ip.trim()); return ipList[0]; } diff --git a/src/users.js b/src/users.js index 831ff09d5..023ddca07 100644 --- a/src/users.js +++ b/src/users.js @@ -511,7 +511,7 @@ async function tryAutoLogin(request) { const userHandles = await getAllUserHandles(); if (userHandles.length === 1) { const user = await storage.getItem(toKey(userHandles[0])); - if (!user.password) { + if (user && !user.password) { request.session.handle = userHandles[0]; return true; } diff --git a/src/util.js b/src/util.js index e1410eee8..e1fd05e82 100644 --- a/src/util.js +++ b/src/util.js @@ -311,9 +311,9 @@ function tryParse(str) { } /** - * Takes a path to a client-accessible file in the `public` folder and converts it to a relative URL segment that the - * client can fetch it from. This involves stripping the `public/` prefix and always using `/` as the separator. - * @param {string} root The root directory of the public folder. + * Takes a path to a client-accessible file in the data folder and converts it to a relative URL segment that the + * client can fetch it from. This involves stripping the data root path prefix and always using `/` as the separator. + * @param {string} root The root directory of the user data folder. * @param {string} inputPath The path to be converted. * @returns The relative URL path from which the client can access the file. */ @@ -350,7 +350,7 @@ function generateTimestamp() { * @param {string} prefix */ function removeOldBackups(prefix) { - const MAX_BACKUPS = 25; + const MAX_BACKUPS = 50; let files = fs.readdirSync(PUBLIC_DIRECTORIES.backups).filter(f => f.startsWith(prefix)); if (files.length > MAX_BACKUPS) {