rixty/SillyTavern
1
0
Fork 0
mirror of https://github.com/SillyTavern/SillyTavern.git synced 2025-06-05 21:59:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'staging' into parser-v2

Browse Source
This commit is contained in:
LenAnderson
2024-04-24 17:55:26 -04:00
parent 1a6e2575a7 01e3964232
commit e7da03d64d
30 changed files with 577 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
.github/workflows/update-docs.yml vendored
View File

@ -1,43 +0,0 @@
name: Update SillyTavern-Docs
on:
push:
branches:
- main
jobs:
update_docs:
runs-on: ubuntu-latest
steps:
- name: Checkout current repository
uses: actions/checkout@v2
- name: Checkout SillyTavern-Docs repository
uses: actions/checkout@v2
with:
repository: SillyTavern/SillyTavern-Docs
path: SillyTavern-Docs
- name: Clone SillyTavern wiki into SillyTavern-Docs/extensions
run: rm -rf SillyTavern-Docs/extensions && git clone https://github.com/SillyTavern/SillyTavern.wiki.git SillyTavern-Docs/extensions && rm -rf SillyTavern-Docs/extensions/.git
- name: Copy files
run: |
cp public/notes/content.md SillyTavern-Docs/guidebook.md
cp faq.md SillyTavern-Docs/faq.md
cp readme.md SillyTavern-Docs/readme.md
cp public/notes/update.md SillyTavern-Docs/update.md
- name: Deploy to external repository
uses: cpina/github-action-push-to-another-repository@main
env:
SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }}
with:
# GitHub Action output files
source-directory: SillyTavern-Docs/
destination-github-username: SillyTavern
destination-repository-name: SillyTavern-Docs
user-email: github-actions[bot]@users.noreply.github.com
user-name: "GitHub Actions"
target-branch: "main"

18
Update-Instructions.txt
View File

@ -33,7 +33,14 @@ If you insist on installing via a zip, here is the tedious process for doing the
2. Unzip it into a folder OUTSIDE of your current ST installation.
3. Do the usual setup procedure for your OS to install the NodeJS requirements.
4. Copy the following files/folders as necessary(*) from your old ST installation:
4a. Updating 1.12.0 and above
Copy the user data directory from your data root into the data root of the new install.
By default: /data/default-user
4a. Migrating from <1.12.0 to >=1.20.0
Copy the following files/folders as necessary(*) from your old ST installation:
- Assets
- Backgrounds
@ -54,16 +61,15 @@ If you insist on installing via a zip, here is the tedious process for doing the
- Worlds
- User
- settings.json
- secrets.json <---- this one is in the base folder, not /public/
- secrets.json <---- This one is in the base folder, not /public/
(*) 'As necessary' = "If you made any custom content related to those folders".
None of the folders are mandatory, so only copy what you need.
**NB: DO NOT COPY THE ENTIRE /PUBLIC/ FOLDER.**
Doing so could break the new install and prevent new features from being present.
Paste those items into the /data/default-user folder of the new install.
5. Paste those items into the /Public/ folder of the new install.
5. Start SillyTavern once again with the method appropriate to your OS, and pray you got it right.
6. Start SillyTavern once again with the method appropriate to your OS, and pray you got it right.
7. If everything shows up, you can safely delete the old ST folder.
6. If everything shows up, you can safely delete the old ST folder.

2
default/config.yaml
View File

@ -9,6 +9,8 @@ port: 8000
# -- SECURITY CONFIGURATION --
# Toggle whitelist mode
whitelistMode: true
# Whitelist will also verify IP in X-Forwarded-For / X-Real-IP headers
enableForwardedWhitelist: true
# Whitelist of allowed IP addresses
whitelist:
- 127.0.0.1

2
public/css/mobile-styles.css
View File

@ -231,9 +231,11 @@
backdrop-filter: blur(calc(var(--SmartThemeBlurStrength) * 2));
}
/*
#right-nav-panel {
padding-right: 15px;
}
*/
#floatingPrompt,
#cfgConfig,

8
public/css/toggle-dependent.css
View File

@ -433,14 +433,6 @@ body.expandMessageActions .mes .mes_buttons .extraMesButtonsHint {
display: none !important;
}
#openai_image_inlining:not(:checked)~#image_inlining_hint {
display: none;
}
#openai_image_inlining:checked~#image_inlining_hint {
display: block;
}
#smooth_streaming:not(:checked)~#smooth_streaming_speed_control {
display: none;
}

11
public/index.html
View File

@ -4982,10 +4982,11 @@
<textarea class="text_pole" rows="1" name="comment" maxlength="5000" data-i18n="[placeholder]Entry Title/Memo" placeholder="Entry Title/Memo"></textarea>
</div>
<!-- <span class="world_entry_form_position_value"></span> -->
<select data-i18n="[title]WI Entry Status:🔵 Constant🟢 Normal❌ Disabled" title="WI Entry Status:&#13;🔵 Constant&#13;🟢 Normal&#13;❌ Disabled" name="entryStateSelector" class="text_pole widthNatural margin0">
<option title="WI Entry Status:&#13;🔵 Constant&#13;🟢 Normal&#13;❌ Disabled" value="constant" data-i18n="[title]WI Entry Status:🔵 Constant🟢 Normal❌ Disabled">🔵</option>
<option title="WI Entry Status:&#13;🔵 Constant&#13;🟢 Normal&#13;❌ Disabled" value="normal" data-i18n="[title]WI Entry Status:🔵 Constant🟢 Normal❌ Disabled">🟢</option>
<option title="WI Entry Status:&#13;🔵 Constant&#13;🟢 Normal&#13;❌ Disabled" value="disabled" data-i18n="[title]WI Entry Status:🔵 Constant🟢 Normal❌ Disabled">❌</option>
<select data-i18n="[title]WI Entry Status:🔵 Constant🟢 Normal🔗 Vectorized❌ Disabled" title="WI Entry Status:&#13;🔵 Constant&#13;🟢 Normal&#13;🔗 Vectorized&#13;❌ Disabled" name="entryStateSelector" class="text_pole widthNatural margin0">
<option value="constant" title="Constant" data-i18n="[title]Constant">🔵</option>
<option value="normal" title="Normal" data-i18n="[title]Normal">🟢</option>
<option value="vectorized" title="Vectorized" data-i18n="[title]Vectorized">🔗</option>
<option value="disabled" title="Disabled" data-i18n="[title]Disabled">❌</option>
</select>
</div>
<div class="WIEnteryHeaderControls flex-container">
@ -6092,4 +6093,4 @@
</script>
</body>
</html>
</html>

3
public/script.js
View File

@ -453,6 +453,7 @@ export const event_types = {
CHARACTER_DUPLICATED: 'character_duplicated',
SMOOTH_STREAM_TOKEN_RECEIVED: 'smooth_stream_token_received',
FILE_ATTACHMENT_DELETED: 'file_attachment_deleted',
WORLDINFO_FORCE_ACTIVATE: 'worldinfo_force_activate',
};
export const eventSource = new EventEmitter();
@ -1844,7 +1845,7 @@ function messageFormatting(mes, ch_name, isSystem, isUser, messageId) {
*/
if (!power_user.allow_name2_display && ch_name && !isUser && !isSystem) {
mes = mes.replace(new RegExp(`(^|\n)${ch_name}:`, 'g'), '$1');
mes = mes.replace(new RegExp(`(^|\n)${escapeRegex(ch_name)}:`, 'g'), '$1');
}
/** @type {any} */

174
public/scripts/chats.js
View File

@ -53,11 +53,11 @@ import { ScraperManager } from './scrapers.js';
* @returns {Promise<string>} Converted file text
*/
const fileSizeLimit = 1024 * 1024 * 10; // 10 MB
const fileSizeLimit = 1024 * 1024 * 100; // 100 MB
const ATTACHMENT_SOURCE = {
GLOBAL: 'global',
CHAT: 'chat',
CHARACTER: 'character',
CHAT: 'chat',
};
/**
@ -592,9 +592,10 @@ async function deleteMessageImage() {
/**
* Deletes file from the server.
* @param {string} url Path to the file on the server
* @param {boolean} [silent=false] If true, do not show error messages
* @returns {Promise<boolean>} True if file was deleted, false otherwise.
*/
async function deleteFileFromServer(url) {
async function deleteFileFromServer(url, silent = false) {
try {
const result = await fetch('/api/files/delete', {
method: 'POST',
@ -602,7 +603,7 @@ async function deleteFileFromServer(url) {
body: JSON.stringify({ path: url }),
});
if (!result.ok) {
if (!result.ok && !silent) {
const error = await result.text();
throw new Error(error);
}
@ -669,6 +670,55 @@ async function editAttachment(attachment, source, callback) {
callback();
}
/**
* Downloads an attachment to the user's device.
* @param {FileAttachment} attachment Attachment to download
*/
async function downloadAttachment(attachment) {
const fileText = attachment.text || (await getFileAttachment(attachment.url));
const blob = new Blob([fileText], { type: 'text/plain' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = attachment.name;
a.click();
URL.revokeObjectURL(url);
}
/**
* Moves a file attachment to a different source.
* @param {FileAttachment} attachment Attachment to moves
* @param {string} source Source of the attachment
* @param {function} callback Success callback
* @returns {Promise<void>} A promise that resolves when the attachment is moved.
*/
async function moveAttachment(attachment, source, callback) {
let selectedTarget = source;
const targets = getAvailableTargets();
const template = $(await renderExtensionTemplateAsync('attachments', 'move-attachment', { name: attachment.name, targets }));
template.find('.moveAttachmentTarget').val(source).on('input', function () {
selectedTarget = String($(this).val());
});
const result = await callGenericPopup(template, POPUP_TYPE.CONFIRM, '', { wide: false, large: false, okButton: 'Move', cancelButton: 'Cancel' });
if (result !== POPUP_RESULT.AFFIRMATIVE) {
console.debug('Move attachment cancelled');
return;
}
if (selectedTarget === source) {
console.debug('Move attachment cancelled: same source and target');
return;
}
const content = await getFileAttachment(attachment.url);
const file = new File([content], attachment.name, { type: 'text/plain' });
await deleteAttachment(attachment, source, () => { }, false);
await uploadFileAttachmentToServer(file, selectedTarget);
callback();
}
/**
* Deletes an attachment from the server and the chat.
* @param {FileAttachment} attachment Attachment to delete
@ -702,7 +752,8 @@ async function deleteAttachment(attachment, source, callback, confirm = true) {
break;
}
await deleteFileFromServer(attachment.url);
const silent = confirm === false;
await deleteFileFromServer(attachment.url, silent);
callback();
}
@ -756,12 +807,15 @@ async function openAttachmentManager() {
for (const attachment of sortedAttachmentList) {
const attachmentTemplate = template.find('.attachmentListItemTemplate .attachmentListItem').clone();
attachmentTemplate.find('.attachmentFileIcon').attr('title', attachment.url);
attachmentTemplate.find('.attachmentListItemName').text(attachment.name);
attachmentTemplate.find('.attachmentListItemSize').text(humanFileSize(attachment.size));
attachmentTemplate.find('.attachmentListItemCreated').text(new Date(attachment.created).toLocaleString());
attachmentTemplate.find('.viewAttachmentButton').on('click', () => openFilePopup(attachment));
attachmentTemplate.find('.editAttachmentButton').on('click', () => editAttachment(attachment, source, renderAttachments));
attachmentTemplate.find('.deleteAttachmentButton').on('click', () => deleteAttachment(attachment, source, renderAttachments));
attachmentTemplate.find('.downloadAttachmentButton').on('click', () => downloadAttachment(attachment));
attachmentTemplate.find('.moveAttachmentButton').on('click', () => moveAttachment(attachment, source, renderAttachments));
template.find(sources[source]).append(attachmentTemplate);
}
}
@ -860,6 +914,50 @@ async function openAttachmentManager() {
template.find('.chatAttachmentsName').text(chatName);
}
function addDragAndDrop() {
$(document.body).on('dragover', '.dialogue_popup', (event) => {
event.preventDefault();
event.stopPropagation();
$(event.target).closest('.dialogue_popup').addClass('dragover');
});
$(document.body).on('dragleave', '.dialogue_popup', (event) => {
event.preventDefault();
event.stopPropagation();
$(event.target).closest('.dialogue_popup').removeClass('dragover');
});
$(document.body).on('drop', '.dialogue_popup', async (event) => {
event.preventDefault();
event.stopPropagation();
$(event.target).closest('.dialogue_popup').removeClass('dragover');
const files = Array.from(event.originalEvent.dataTransfer.files);
let selectedTarget = ATTACHMENT_SOURCE.GLOBAL;
const targets = getAvailableTargets();
const targetSelectTemplate = $(await renderExtensionTemplateAsync('attachments', 'files-dropped', { count: files.length, targets: targets }));
targetSelectTemplate.find('.droppedFilesTarget').on('input', function () {
selectedTarget = String($(this).val());
});
const result = await callGenericPopup(targetSelectTemplate, POPUP_TYPE.CONFIRM, '', { wide: false, large: false, okButton: 'Upload', cancelButton: 'Cancel' });
if (result !== POPUP_RESULT.AFFIRMATIVE) {
console.log('File upload cancelled');
return;
}
for (const file of files) {
await uploadFileAttachmentToServer(file, selectedTarget);
}
renderAttachments();
});
}
function removeDragAndDrop() {
$(document.body).off('dragover', '.shadow_popup');
$(document.body).off('dragleave', '.shadow_popup');
$(document.body).off('drop', '.shadow_popup');
}
let sortField = localStorage.getItem('DataBank_sortField') || 'created';
let sortOrder = localStorage.getItem('DataBank_sortOrder') || 'desc';
let filterString = '';
@ -883,10 +981,34 @@ async function openAttachmentManager() {
});
const cleanupFn = await renderButtons();
await verifyAttachments();
await renderAttachments();
addDragAndDrop();
await callGenericPopup(template, POPUP_TYPE.TEXT, '', { wide: true, large: true, okButton: 'Close' });
cleanupFn();
removeDragAndDrop();
}
/**
* Gets a list of available targets for attachments.
* @returns {string[]} List of available targets
*/
function getAvailableTargets() {
const targets = Object.values(ATTACHMENT_SOURCE);
const isNotCharacter = this_chid === undefined || selected_group;
const isNotInChat = getCurrentChatId() === undefined;
if (isNotCharacter) {
targets.splice(targets.indexOf(ATTACHMENT_SOURCE.CHARACTER), 1);
}
if (isNotInChat) {
targets.splice(targets.indexOf(ATTACHMENT_SOURCE.CHAT), 1);
}
return targets;
}
/**
@ -1039,6 +1161,48 @@ export function getDataBankAttachmentsForSource(source) {
}
}
/**
* Verifies all attachments in the Data Bank.
* @returns {Promise<void>} A promise that resolves when attachments are verified.
*/
async function verifyAttachments() {
for (const source of Object.values(ATTACHMENT_SOURCE)) {
await verifyAttachmentsForSource(source);
}
}
/**
* Verifies all attachments for a specific source.
* @param {string} source Attachment source
* @returns {Promise<void>} A promise that resolves when attachments are verified.
*/
async function verifyAttachmentsForSource(source) {
try {
const attachments = getDataBankAttachmentsForSource(source);
const urls = attachments.map(a => a.url);
const response = await fetch('/api/files/verify', {
method: 'POST',
headers: getRequestHeaders(),
body: JSON.stringify({ urls }),
});
if (!response.ok) {
const error = await response.text();
throw new Error(error);
}
const verifiedUrls = await response.json();
for (const attachment of attachments) {
if (verifiedUrls[attachment.url] === false) {
console.log('Deleting orphaned attachment', attachment);
await deleteAttachment(attachment, source, () => { }, false);
}
}
} catch (error) {
console.error('Attachment verification failed', error);
}
}
/**
* Registers a file converter function.
* @param {string} mimeType MIME type

8
public/scripts/extensions/attachments/files-dropped.html Normal file
View File

@ -0,0 +1,8 @@
<div class="flex-container justifyCenter alignItemsBaseline">
<span>Save <span class="droppedFilesCount">{{count}}</span> file(s) to...</span>
<select class="droppedFilesTarget">
{{#each targets}}
<option value="{{this}}">{{this}}</option>
{{/each}}
</select>
</div>

3
public/scripts/extensions/attachments/index.js
View File

@ -1,6 +1,9 @@
import { renderExtensionTemplateAsync } from '../../extensions.js';
import { registerSlashCommand } from '../../slash-commands.js';
jQuery(async () => {
const buttons = await renderExtensionTemplateAsync('attachments', 'buttons', {});
$('#extensionsMenu').prepend(buttons);
registerSlashCommand('db', () => document.getElementById('manageAttachments')?.click(), ['databank', 'data-bank'], '– open the data bank', true, true);
});

11
public/scripts/extensions/attachments/manager.html
View File

@ -7,8 +7,13 @@
<div data-i18n="These files will be available for extensions that support attachments (e.g. Vector Storage).">
These files will be available for extensions that support attachments (e.g. Vector Storage).
</div>
<div data-i18n="Supported file types: Plain Text, PDF, Markdown, HTML, EPUB." class="marginTopBot5">
Supported file types: Plain Text, PDF, Markdown, HTML, EPUB.
<div class="marginTopBot5">
<span data-i18n="Supported file types: Plain Text, PDF, Markdown, HTML, EPUB." >
Supported file types: Plain Text, PDF, Markdown, HTML, EPUB.
</span>
<span data-i18n="Drag and drop files here to upload.">
Drag and drop files here to upload.
</span>
</div>
<div class="flex-container marginTopBot5">
<input type="search" id="attachmentSearch" class="attachmentSearch text_pole margin0 flex1" placeholder="Search...">
@ -102,7 +107,9 @@
<small class="attachmentListItemCreated"></small>
<small class="attachmentListItemSize"></small>
<div class="viewAttachmentButton right_menu_button fa-solid fa-magnifying-glass" title="View attachment content"></div>
<div class="moveAttachmentButton right_menu_button fa-solid fa-arrows-alt" title="Move attachment"></div>
<div class="editAttachmentButton right_menu_button fa-solid fa-pencil" title="Edit attachment"></div>
<div class="downloadAttachmentButton right_menu_button fa-solid fa-download" title="Download attachment"></div>
<div class="deleteAttachmentButton right_menu_button fa-solid fa-trash" title="Delete attachment"></div>
</div>
</div>

2
public/scripts/extensions/attachments/manifest.json
View File

@ -1,5 +1,5 @@
{
"display_name": "Chat Attachments",
"display_name": "Data Bank (Chat Attachments)",
"loading_order": 3,
"requires": [],
"optional": [],

8
public/scripts/extensions/attachments/move-attachment.html Normal file
View File

@ -0,0 +1,8 @@
<div class="flex-container justifyCenter alignItemsBaseline">
<span>Move <strong class="moveAttachmentName">{{name}}</strong> to...</span>
<select class="moveAttachmentTarget">
{{#each targets}}
<option value="{{this}}">{{this}}</option>
{{/each}}
</select>
</div>

2
public/scripts/extensions/expressions/settings.html
View File

@ -78,7 +78,7 @@
<span>Remove all image overrides</span>
</div>
</div>
<p class="hint"><b>Hint:</b> <i>Create new folder in the <b>public/characters/</b> folder and name it as the name of the character.
<p class="hint"><b>Hint:</b> <i>Create new folder in the <b>/characters/</b> folder of your user data directory and name it as the name of the character.
Put images with expressions there. File names should follow the pattern: <tt>[expression_label].[image_format]</tt></i></p>
</div>
</div>

9
public/scripts/extensions/quick-reply/index.js
View File

@ -1,4 +1,4 @@
import { chat_metadata, eventSource, event_types, getRequestHeaders } from '../../../script.js';
import { chat, chat_metadata, eventSource, event_types, getRequestHeaders } from '../../../script.js';
import { extension_settings } from '../../extensions.js';
import { QuickReplyApi } from './api/QuickReplyApi.js';
import { AutoExecuteHandler } from './src/AutoExecuteHandler.js';
@ -240,7 +240,12 @@ const onUserMessage = async () => {
};
eventSource.on(event_types.USER_MESSAGE_RENDERED, (...args)=>executeIfReadyElseQueue(onUserMessage, args));
const onAiMessage = async () => {
const onAiMessage = async (messageId) => {
if (['...'].includes(chat[messageId]?.mes)) {
log('QR auto-execution suppressed for swiped message');
return;
}
await autoExec.handleAi();
};
eventSource.on(event_types.CHARACTER_MESSAGE_RENDERED, (...args)=>executeIfReadyElseQueue(onAiMessage, args));

4
public/scripts/extensions/stable-diffusion/index.js
View File

@ -1657,6 +1657,10 @@ async function loadNovelModels() {
value: 'safe-diffusion',
text: 'NAI Diffusion Anime V1 (Curated)',
},
{
value: 'nai-diffusion-furry-3',
text: 'NAI Diffusion Furry V3',
},
{
value: 'nai-diffusion-furry',
text: 'NAI Diffusion Furry',

147
public/scripts/extensions/vectors/index.js
View File

@ -23,6 +23,7 @@ import { collapseNewlines } from '../../power-user.js';
import { SECRET_KEYS, secret_state, writeSecret } from '../../secrets.js';
import { getDataBankAttachments, getFileAttachment } from '../../chats.js';
import { debounce, getStringHash as calculateHash, waitUntilCondition, onlyUnique, splitRecursive } from '../../utils.js';
import { getSortedEntries } from '../../world-info.js';
const MODULE_NAME = 'vectors';
@ -66,6 +67,11 @@ const settings = {
file_position_db: extension_prompt_types.IN_PROMPT,
file_depth_db: 4,
file_depth_role_db: extension_prompt_roles.SYSTEM,
// For World Info
enabled_world_info: false,
enabled_for_all: false,
max_entries: 5,
};
const moduleWorker = new ModuleWorkerWrapper(synchronizeChat);
@ -281,8 +287,10 @@ async function synchronizeChat(batchSize = 5) {
}
}
// Cache object for storing hash values
const hashCache = {};
/**
* @type {Map<string, number>} Cache object for storing hash values
*/
const hashCache = new Map();
/**
* Gets the hash value for a given string
@ -291,15 +299,15 @@ const hashCache = {};
*/
function getStringHash(str) {
// Check if the hash is already in the cache
if (Object.hasOwn(hashCache, str)) {
return hashCache[str];
if (hashCache.has(str)) {
return hashCache.get(str);
}
// Calculate the hash value
const hash = calculateHash(str);
// Store the hash in the cache
hashCache[str] = hash;
hashCache.set(str, hash);
return hash;
}
@ -472,6 +480,10 @@ async function rearrangeChat(chat) {
await processFiles(chat);
}
if (settings.enabled_world_info) {
await activateWorldInfo(chat);
}
if (!settings.enabled_chats) {
return;
}
@ -845,6 +857,7 @@ async function purgeVectorIndex(collectionId) {
function toggleSettings() {
$('#vectors_files_settings').toggle(!!settings.enabled_files);
$('#vectors_chats_settings').toggle(!!settings.enabled_chats);
$('#vectors_world_info_settings').toggle(!!settings.enabled_world_info);
$('#together_vectorsModel').toggle(settings.source === 'togetherai');
$('#openai_vectorsModel').toggle(settings.source === 'openai');
$('#cohere_vectorsModel').toggle(settings.source === 'cohere');
@ -934,6 +947,111 @@ async function onPurgeFilesClick() {
}
}
async function activateWorldInfo(chat) {
if (!settings.enabled_world_info) {
console.debug('Vectors: Disabled for World Info');
return;
}
const entries = await getSortedEntries();
if (!Array.isArray(entries) || entries.length === 0) {
console.debug('Vectors: No WI entries found');
return;
}
// Group entries by "world" field
const groupedEntries = {};
for (const entry of entries) {
// Skip orphaned entries. Is it even possible?
if (!entry.world) {
console.debug('Vectors: Skipped orphaned WI entry', entry);
continue;
}
// Skip disabled entries
if (entry.disable) {
console.debug('Vectors: Skipped disabled WI entry', entry);
continue;
}
// Skip entries without content
if (!entry.content) {
console.debug('Vectors: Skipped WI entry without content', entry);
continue;
}
// Skip non-vectorized entries
if (!entry.vectorized && !settings.enabled_for_all) {
console.debug('Vectors: Skipped non-vectorized WI entry', entry);
continue;
}
if (!Object.hasOwn(groupedEntries, entry.world)) {
groupedEntries[entry.world] = [];
}
groupedEntries[entry.world].push(entry);
}
const collectionIds = [];
if (Object.keys(groupedEntries).length === 0) {
console.debug('Vectors: No WI entries to synchronize');
return;
}
// Synchronize collections
for (const world in groupedEntries) {
const collectionId = `world_${getStringHash(world)}`;
const hashesInCollection = await getSavedHashes(collectionId);
const newEntries = groupedEntries[world].filter(x => !hashesInCollection.includes(getStringHash(x.content)));
const deletedHashes = hashesInCollection.filter(x => !groupedEntries[world].some(y => getStringHash(y.content) === x));
if (newEntries.length > 0) {
console.log(`Vectors: Found ${newEntries.length} new WI entries for world ${world}`);
await insertVectorItems(collectionId, newEntries.map(x => ({ hash: getStringHash(x.content), text: x.content, index: x.uid })));
}
if (deletedHashes.length > 0) {
console.log(`Vectors: Deleted ${deletedHashes.length} old hashes for world ${world}`);
await deleteVectorItems(collectionId, deletedHashes);
}
collectionIds.push(collectionId);
}
// Perform a multi-query
const queryText = await getQueryText(chat);
if (queryText.length === 0) {
console.debug('Vectors: No text to query for WI');
return;
}
const queryResults = await queryMultipleCollections(collectionIds, queryText, settings.max_entries);
const activatedHashes = Object.values(queryResults).flatMap(x => x.hashes).filter(onlyUnique);
const activatedEntries = [];
// Activate entries found in the query results
for (const entry of entries) {
const hash = getStringHash(entry.content);
if (activatedHashes.includes(hash)) {
activatedEntries.push(entry);
}
}
if (activatedEntries.length === 0) {
console.debug('Vectors: No activated WI entries found');
return;
}
console.log(`Vectors: Activated ${activatedEntries.length} WI entries`, activatedEntries);
await eventSource.emit(event_types.WORLDINFO_FORCE_ACTIVATE, activatedEntries);
}
jQuery(async () => {
if (!extension_settings.vectors) {
extension_settings.vectors = settings;
@ -1134,6 +1252,25 @@ jQuery(async () => {
saveSettingsDebounced();
});
$('#vectors_enabled_world_info').prop('checked', settings.enabled_world_info).on('input', () => {
settings.enabled_world_info = !!$('#vectors_enabled_world_info').prop('checked');
Object.assign(extension_settings.vectors, settings);
saveSettingsDebounced();
toggleSettings();
});
$('#vectors_enabled_for_all').prop('checked', settings.enabled_for_all).on('input', () => {
settings.enabled_for_all = !!$('#vectors_enabled_for_all').prop('checked');
Object.assign(extension_settings.vectors, settings);
saveSettingsDebounced();
});
$('#vectors_max_entries').val(settings.max_entries).on('input', () => {
settings.max_entries = Number($('#vectors_max_entries').val());
Object.assign(extension_settings.vectors, settings);
saveSettingsDebounced();
});
const validSecret = !!secret_state[SECRET_KEYS.NOMICAI];
const placeholder = validSecret ? '✔️ Key saved' : '❌ Missing key';
$('#api_key_nomicai').attr('placeholder', placeholder);

40
public/scripts/extensions/vectors/settings.html
View File

@ -97,6 +97,46 @@
<hr>
<h4>
World Info settings
</h4>
<label class="checkbox_label" for="vectors_enabled_world_info" title="Enable activation of World Info entries based on vector similarity.">
<input id="vectors_enabled_world_info" type="checkbox" class="checkbox">
Enabled for World Info
</label>
<div id="vectors_world_info_settings" class="marginTopBot5">
<div class="flex-container">
<label for="vectors_enabled_for_all" class="checkbox_label">
<input id="vectors_enabled_for_all" type="checkbox" />
<span>Enabled for all entries</span>
</label>
<ul class="margin0">
<li>
<small>Checked: all entries except ❌ status can be activated.</small>
</li>
<li>
<small>Unchecked: only entries with 🔗 status can be activated.</small>
</li>
</ul>
</div>
<div class="flex-container">
<div class="flex1">
<!-- Vacant for future use -->
</div>
<div class="flex1" title="Maximum number of entries to be activated">
<label for="vectors_max_entries" >
<small>Max Entries</small>
</label>
<input id="vectors_max_entries" type="number" class="text_pole widthUnset" min="1" max="9999" />
</div>
<div class="flex1">
<!-- Vacant for future use -->
</div>
</div>
</div>
<h4>
File vectorization settings
</h4>

77
public/scripts/world-info.js
View File

@ -82,6 +82,11 @@ class WorldInfoBuffer {
/** @typedef {{scanDepth?: number, caseSensitive?: boolean, matchWholeWords?: boolean}} WIScanEntry The entry that triggered the scan */
// End typedef area
/**
* @type {object[]} Array of entries that need to be activated no matter what
*/
static externalActivations = [];
/**
* @type {string[]} Array of messages sorted by ascending depth
*/
@ -220,6 +225,23 @@ class WorldInfoBuffer {
getDepth() {
return world_info_depth + this.#skew;
}
/**
* Check if the current entry is externally activated.
* @param {object} entry WI entry to check
* @returns {boolean} True if the entry is forcefully activated
*/
isExternallyActivated(entry) {
// Entries could be copied with structuredClone, so we need to compare them by string representation
return WorldInfoBuffer.externalActivations.some(x => JSON.stringify(x) === JSON.stringify(entry));
}
/**
* Clears the force activations buffer.
*/
cleanExternalActivations() {
WorldInfoBuffer.externalActivations.splice(0, WorldInfoBuffer.externalActivations.length);
}
}
export function getWorldInfoSettings() {
@ -362,6 +384,10 @@ function setWorldInfoSettings(settings, data) {
$('.chat_lorebook_button').toggleClass('world_set', hasWorldInfo);
});
eventSource.on(event_types.WORLDINFO_FORCE_ACTIVATE, (entries) => {
WorldInfoBuffer.externalActivations.push(...entries);
});
// Add slash commands
registerWorldInfoSlashCommands();
}
@ -564,6 +590,7 @@ function registerWorldInfoSlashCommands() {
return '';
}
registerSlashCommand('world', onWorldInfoChange, [], '<span class="monospace">[optional state=off|toggle] [optional silent=true] (optional name)</span> – sets active World, or unsets if no args provided, use <code>state=off</code> and <code>state=toggle</code> to deactivate or toggle a World, use <code>silent=true</code> to suppress toast messages', true, true);
registerSlashCommand('getchatbook', getChatBookCallback, ['getchatlore', 'getchatwi'], '– get a name of the chat-bound lorebook or create a new one if was unbound, and pass it down the pipe', true, true);
registerSlashCommand('findentry', findBookEntryCallback, ['findlore', 'findwi'], '<span class="monospace">(file=bookName field=field [texts])</span> – find a UID of the record from the specified book using the fuzzy match of a field value (default: key) and pass it down the pipe, e.g. <tt>/findentry file=chatLore field=key Shadowfang</tt>', true, true);
registerSlashCommand('getentryfield', getEntryFieldCallback, ['getlorefield', 'getwifield'], '<span class="monospace">(file=bookName field=field [UID])</span> – get a field value (default: content) of the record with the UID from the specified book and pass it down the pipe, e.g. <tt>/getentryfield file=chatLore field=content 123</tt>', true, true);
@ -964,6 +991,7 @@ const originalDataKeyMap = {
'caseSensitive': 'extensions.case_sensitive',
'scanDepth': 'extensions.scan_depth',
'automationId': 'extensions.automation_id',
'vectorized': 'extensions.vectorized',
};
function setOriginalDataValue(data, uid, key, value) {
@ -1071,6 +1099,16 @@ function getWorldEntry(name, data, entry) {
);
}
// Verify names to exist in the system
if (data.entries[uid]?.characterFilter?.names?.length > 0) {
for (const name of [...data.entries[uid].characterFilter.names]) {
if (!getContext().characters.find(x => x.avatar.replace(/\.[^/.]+$/, '') === name)) {
console.warn(`World Info: Character ${name} not found. Removing from the entry filter.`, entry);
data.entries[uid].characterFilter.names = data.entries[uid].characterFilter.names.filter(x => x !== name);
}
}
}
setOriginalDataValue(data, uid, 'character_filter', data.entries[uid].characterFilter);
saveWorldInfo(name, data);
});
@ -1454,22 +1492,37 @@ function getWorldEntry(name, data, entry) {
case 'constant':
data.entries[uid].constant = true;
data.entries[uid].disable = false;
data.entries[uid].vectorized = false;
setOriginalDataValue(data, uid, 'enabled', true);
setOriginalDataValue(data, uid, 'constant', true);
setOriginalDataValue(data, uid, 'extensions.vectorized', false);
template.removeClass('disabledWIEntry');
break;
case 'normal':
data.entries[uid].constant = false;
data.entries[uid].disable = false;
data.entries[uid].vectorized = false;
setOriginalDataValue(data, uid, 'enabled', true);
setOriginalDataValue(data, uid, 'constant', false);
setOriginalDataValue(data, uid, 'extensions.vectorized', false);
template.removeClass('disabledWIEntry');
break;
case 'vectorized':
data.entries[uid].constant = false;
data.entries[uid].disable = false;
data.entries[uid].vectorized = true;
setOriginalDataValue(data, uid, 'enabled', true);
setOriginalDataValue(data, uid, 'constant', false);
setOriginalDataValue(data, uid, 'extensions.vectorized', true);
template.removeClass('disabledWIEntry');
break;
case 'disabled':
data.entries[uid].constant = false;
data.entries[uid].disable = true;
data.entries[uid].vectorized = false;
setOriginalDataValue(data, uid, 'enabled', false);
setOriginalDataValue(data, uid, 'constant', false);
setOriginalDataValue(data, uid, 'extensions.vectorized', false);
template.addClass('disabledWIEntry');
break;
}
@ -1480,6 +1533,8 @@ function getWorldEntry(name, data, entry) {
const entryState = function () {
if (entry.constant === true) {
return 'constant';
} else if (entry.vectorized === true) {
return 'vectorized';
} else if (entry.disable === true) {
return 'disabled';
} else {
@ -1719,6 +1774,7 @@ const newEntryTemplate = {
comment: '',
content: '',
constant: false,
vectorized: false,
selective: true,
selectiveLogic: world_info_logic.AND_ANY,
addMemo: false,
@ -1925,7 +1981,7 @@ async function getCharacterLore() {
}
const data = await loadWorldInfoData(worldName);
const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]) : [];
const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x => ({ ...x, world: worldName })) : [];
entries = entries.concat(newEntries);
}
@ -1941,7 +1997,7 @@ async function getGlobalLore() {
let entries = [];
for (const worldName of selected_world_info) {
const data = await loadWorldInfoData(worldName);
const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]) : [];
const newEntries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x => ({ ...x, world: worldName })) : [];
entries = entries.concat(newEntries);
}
@ -1963,14 +2019,14 @@ async function getChatLore() {
}
const data = await loadWorldInfoData(chatWorld);
const entries = data ? Object.keys(data.entries).map((x) => data.entries[x]) : [];
const entries = data ? Object.keys(data.entries).map((x) => data.entries[x]).map(x => ({ ...x, world: chatWorld })) : [];
console.debug(`Chat lore has ${entries.length} entries`);
return entries;
}
async function getSortedEntries() {
export async function getSortedEntries() {
try {
const globalLore = await getGlobalLore();
const characterLore = await getCharacterLore();
@ -2098,7 +2154,7 @@ async function checkWorldInfo(chat, maxContext) {
continue;
}
if (entry.constant) {
if (entry.constant || buffer.isExternallyActivated(entry)) {
entry.content = substituteParams(entry.content);
activatedNow.add(entry);
continue;
@ -2295,6 +2351,8 @@ async function checkWorldInfo(chat, maxContext) {
context.setExtensionPrompt(NOTE_MODULE_NAME, ANWithWI, chat_metadata[metadata_keys.position], chat_metadata[metadata_keys.depth], extension_settings.note.allowWIScan, chat_metadata[metadata_keys.role]);
}
buffer.cleanExternalActivations();
return { worldInfoBefore, worldInfoAfter, WIDepthEntries, allActivatedEntries };
}
@ -2381,6 +2439,7 @@ function convertAgnaiMemoryBook(inputObj) {
content: entry.entry,
constant: false,
selective: false,
vectorized: false,
selectiveLogic: world_info_logic.AND_ANY,
order: entry.weight,
position: 0,
@ -2415,6 +2474,7 @@ function convertRisuLorebook(inputObj) {
content: entry.content,
constant: entry.alwaysActive,
selective: entry.selective,
vectorized: false,
selectiveLogic: world_info_logic.AND_ANY,
order: entry.insertorder,
position: world_info_position.before,
@ -2454,6 +2514,7 @@ function convertNovelLorebook(inputObj) {
content: entry.text,
constant: false,
selective: false,
vectorized: false,
selectiveLogic: world_info_logic.AND_ANY,
order: entry.contextConfig?.budgetPriority ?? 0,
position: 0,
@ -2510,6 +2571,7 @@ function convertCharacterBook(characterBook) {
matchWholeWords: entry.extensions?.match_whole_words ?? null,
automationId: entry.extensions?.automation_id ?? '',
role: entry.extensions?.role ?? extension_prompt_roles.SYSTEM,
vectorized: entry.extensions?.vectorized ?? false,
};
});
@ -2785,11 +2847,6 @@ function assignLorebookToChat() {
jQuery(() => {
$(document).ready(function () {
registerSlashCommand('world', onWorldInfoChange, [], '<span class="monospace">[optional state=off|toggle] [optional silent=true] (optional name)</span> – sets active World, or unsets if no args provided, use <code>state=off</code> and <code>state=toggle</code> to deactivate or toggle a World, use <code>silent=true</code> to suppress toast messages', true, true);
});
$('#world_info').on('mousedown change', async function (e) {
// If there's no world names, don't do anything
if (world_names.length === 0) {

5
public/style.css
View File

@ -2780,6 +2780,11 @@ grammarly-extension {
}
}
.dialogue_popup.dragover {
filter: brightness(1.1) saturate(1.1);
outline: 3px dashed var(--SmartThemeBorderColor);
}
#bgtest {
display: none;
width: 100vw;

10
server.js
View File

@ -263,10 +263,14 @@ app.get('/login', async (request, response) => {
return response.redirect('/');
}
const autoLogin = await userModule.tryAutoLogin(request);
try {
const autoLogin = await userModule.tryAutoLogin(request);
if (autoLogin) {
return response.redirect('/');
if (autoLogin) {
return response.redirect('/');
}
} catch (error) {
console.error('Error during auto-login:', error);
}
return response.sendFile('login.html', { root: path.join(process.cwd(), 'public') });

2
src/endpoints/assets.js
View File

@ -56,6 +56,8 @@ function validateAssetFileName(inputFilename) {
* @returns {string[]} - The array of files
*/
function getFiles(dir, files = []) {
if (!fs.existsSync(dir)) return files;
// Get an array of all files and directories in the passed directory using fs.readdirSync
const fileList = fs.readdirSync(dir, { withFileTypes: true });
// Create the full path of the file/directory by concatenating the passed directory and file/directory name

2
src/endpoints/backends/text-completions.js
View File

@ -325,7 +325,7 @@ router.post('/generate', jsonParser, async function (request, response) {
// Map InfermaticAI response to OAI completions format
if (apiType === TEXTGEN_TYPES.INFERMATICAI) {
data['choices'] = (data?.choices || []).map(choice => ({ text: choice.message.content }));
data['choices'] = (data?.choices || []).map(choice => ({ text: choice?.message?.content || choice.text }));
}
return response.send(data);

3
src/endpoints/characters.js
View File

@ -439,6 +439,7 @@ function convertWorldInfoToCharacterBook(name, entries) {
case_sensitive: entry.caseSensitive ?? null,
automation_id: entry.automationId ?? '',
role: entry.role ?? 0,
vectorized: entry.vectorized ?? false,
},
};
@ -1097,7 +1098,7 @@ router.post('/export', jsonParser, async function (request, response) {
const fileContent = await fsPromises.readFile(filename);
const contentType = mime.lookup(filename) || 'image/png';
response.setHeader('Content-Type', contentType);
response.setHeader('Content-Disposition', `attachment; filename=${path.basename(filename)}`);
response.setHeader('Content-Disposition', `attachment; filename="${encodeURI(path.basename(filename))}"`);
return response.send(fileContent);
}
case 'json': {

16
src/endpoints/classify.js
View File

@ -5,7 +5,10 @@ const TASK = 'text-classification';
const router = express.Router();
const cacheObject = {};
/**
* @type {Map<string, object>} Cache for classification results
*/
const cacheObject = new Map();
router.post('/labels', jsonParser, async (req, res) => {
try {
@ -23,15 +26,20 @@ router.post('/', jsonParser, async (req, res) => {
try {
const { text } = req.body;
/**
* Get classification result for a given text
* @param {string} text Text to classify
* @returns {Promise<object>} Classification result
*/
async function getResult(text) {
if (Object.hasOwn(cacheObject, text)) {
return cacheObject[text];
if (cacheObject.has(text)) {
return cacheObject.get(text);
} else {
const module = await import('../transformers.mjs');
const pipe = await module.default.getPipeline(TASK);
const result = await pipe(text, { topk: 5 });
result.sort((a, b) => b.score - a.score);
cacheObject[text] = result;
cacheObject.set(text, result);
return result;
}
}

25
src/endpoints/files.js
View File

@ -57,4 +57,29 @@ router.post('/delete', jsonParser, async (request, response) => {
}
});
router.post('/verify', jsonParser, async (request, response) => {
try {
if (!Array.isArray(request.body.urls)) {
return response.status(400).send('No URLs specified');
}
const verified = {};
for (const url of request.body.urls) {
const pathToVerify = path.join(request.user.directories.root, url);
if (!pathToVerify.startsWith(request.user.directories.files)) {
console.debug(`File verification: Invalid path: ${pathToVerify}`);
continue;
}
const fileExists = fs.existsSync(pathToVerify);
verified[url] = fileExists;
}
return response.send(verified);
} catch (error) {
console.log(error);
return response.sendStatus(500);
}
});
module.exports = { router };

28
src/endpoints/settings.js
View File

@ -1,6 +1,7 @@
const fs = require('fs');
const path = require('path');
const express = require('express');
const _ = require('lodash');
const writeFileAtomicSync = require('write-file-atomic').sync;
const { PUBLIC_DIRECTORIES, SETTINGS_FILE } = require('../constants');
const { getConfigValue, generateTimestamp, removeOldBackups } = require('../util');
@ -10,6 +11,32 @@ const { getAllUserHandles, getUserDirectories } = require('../users');
const ENABLE_EXTENSIONS = getConfigValue('enableExtensions', true);
const ENABLE_ACCOUNTS = getConfigValue('enableUserAccounts', false);
// 10 minutes
const AUTOSAVE_INTERVAL = 10 * 60 * 1000;
/**
* Map of functions to trigger settings autosave for a user.
* @type {Map<string, function>}
*/
const AUTOSAVE_FUNCTIONS = new Map();
/**
* Triggers autosave for a user every 10 minutes.
* @param {string} handle User handle
* @returns {void}
*/
function triggerAutoSave(handle) {
if (!AUTOSAVE_FUNCTIONS.has(handle)) {
const throttledAutoSave = _.throttle(() => backupUserSettings(handle), AUTOSAVE_INTERVAL);
AUTOSAVE_FUNCTIONS.set(handle, throttledAutoSave);
}
const functionToCall = AUTOSAVE_FUNCTIONS.get(handle);
if (functionToCall) {
functionToCall();
}
}
/**
* Reads and parses files from a directory.
* @param {string} directoryPath Path to the directory
@ -121,6 +148,7 @@ router.post('/save', jsonParser, function (request, response) {
try {
const pathToSettings = path.join(request.user.directories.root, SETTINGS_FILE);
writeFileAtomicSync(pathToSettings, JSON.stringify(request.body, null, 4), 'utf8');
triggerAutoSave(request.user.profile.handle);
response.send({ result: 'ok' });
} catch (err) {
console.log(err);

9
src/middleware/whitelist.js
View File

@ -6,6 +6,7 @@ const { getIpFromRequest } = require('../express-common');
const { color, getConfigValue } = require('../util');
const whitelistPath = path.join(process.cwd(), './whitelist.txt');
const enableForwardedWhitelist = getConfigValue('enableForwardedWhitelist', false);
let whitelist = getConfigValue('whitelist', []);
let knownIPs = new Set();
@ -24,14 +25,18 @@ if (fs.existsSync(whitelistPath)) {
* @returns {string|undefined} The client IP address
*/
function getForwardedIp(req) {
if (!enableForwardedWhitelist) {
return undefined;
}
// Check if X-Real-IP is available
if (req.headers['x-real-ip']) {
return req.headers['x-real-ip'];
return req.headers['x-real-ip'].toString();
}
// Check for X-Forwarded-For and parse if available
if (req.headers['x-forwarded-for']) {
const ipList = req.headers['x-forwarded-for'].split(',').map(ip => ip.trim());
const ipList = req.headers['x-forwarded-for'].toString().split(',').map(ip => ip.trim());
return ipList[0];
}

2
src/users.js
View File

@ -511,7 +511,7 @@ async function tryAutoLogin(request) {
const userHandles = await getAllUserHandles();
if (userHandles.length === 1) {
const user = await storage.getItem(toKey(userHandles[0]));
if (!user.password) {
if (user && !user.password) {
request.session.handle = userHandles[0];
return true;
}

8
src/util.js
View File

@ -311,9 +311,9 @@ function tryParse(str) {
}
/**
* Takes a path to a client-accessible file in the `public` folder and converts it to a relative URL segment that the
* client can fetch it from. This involves stripping the `public/` prefix and always using `/` as the separator.
* @param {string} root The root directory of the public folder.
* Takes a path to a client-accessible file in the data folder and converts it to a relative URL segment that the
* client can fetch it from. This involves stripping the data root path prefix and always using `/` as the separator.
* @param {string} root The root directory of the user data folder.
* @param {string} inputPath The path to be converted.
* @returns The relative URL path from which the client can access the file.
*/
@ -350,7 +350,7 @@ function generateTimestamp() {
* @param {string} prefix
*/
function removeOldBackups(prefix) {
const MAX_BACKUPS = 25;
const MAX_BACKUPS = 50;
let files = fs.readdirSync(PUBLIC_DIRECTORIES.backups).filter(f => f.startsWith(prefix));
if (files.length > MAX_BACKUPS) {