Merge branch 'staging' into feat/xtc

index.d.ts

@@ -9,6 +9,11 @@ declare global {
             };
         }
     }
+
+    /**
+     * The root directory for user data.
+     */
+    var DATA_ROOT: string;
 }
 
 declare module 'express-session' {

package-lock.json

@@ -27,6 +27,7 @@
                 "google-translate-api-browser": "^3.0.1",
                 "he": "^1.2.0",
                 "helmet": "^7.1.0",
+                "iconv-lite": "^0.6.3",
                 "ip-matching": "^2.1.2",
                 "ipaddr.js": "^2.0.1",
                 "jimp": "^0.22.10",
@@ -42,7 +43,7 @@
                 "rate-limiter-flexible": "^5.0.0",
                 "response-time": "^2.3.2",
                 "sanitize-filename": "^1.6.3",
-                "sillytavern-transformers": "^2.14.6",
+                "sillytavern-transformers": "2.14.6",
                 "simple-git": "^3.19.1",
                 "tiktoken": "^1.0.15",
                 "vectra": "^0.2.2",
@@ -1492,6 +1493,18 @@
                 "node": ">= 0.8"
             }
         },
+        "node_modules/body-parser/node_modules/iconv-lite": {
+            "version": "0.4.24",
+            "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
+            "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+            "license": "MIT",
+            "dependencies": {
+                "safer-buffer": ">= 2.1.2 < 3"
+            },
+            "engines": {
+                "node": ">=0.10.0"
+            }
+        },
         "node_modules/boolbase": {
             "version": "1.0.0",
             "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
@@ -3282,12 +3295,12 @@
             }
         },
         "node_modules/iconv-lite": {
-            "version": "0.4.24",
+            "version": "0.6.3",
-            "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
+            "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
-            "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+            "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
             "license": "MIT",
             "dependencies": {
-                "safer-buffer": ">= 2.1.2 < 3"
+                "safer-buffer": ">= 2.1.2 < 3.0.0"
             },
             "engines": {
                 "node": ">=0.10.0"
@@ -4618,6 +4631,18 @@
                 "node": ">= 0.8"
             }
         },
+        "node_modules/raw-body/node_modules/iconv-lite": {
+            "version": "0.4.24",
+            "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
+            "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+            "license": "MIT",
+            "dependencies": {
+                "safer-buffer": ">= 2.1.2 < 3"
+            },
+            "engines": {
+                "node": ">=0.10.0"
+            }
+        },
         "node_modules/readable-stream": {
             "version": "2.3.8",
             "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",

package.json

@@ -17,6 +17,7 @@
         "google-translate-api-browser": "^3.0.1",
         "he": "^1.2.0",
         "helmet": "^7.1.0",
+        "iconv-lite": "^0.6.3",
         "ip-matching": "^2.1.2",
         "ipaddr.js": "^2.0.1",
         "jimp": "^0.22.10",
@@ -32,7 +33,7 @@
         "rate-limiter-flexible": "^5.0.0",
         "response-time": "^2.3.2",
         "sanitize-filename": "^1.6.3",
-        "sillytavern-transformers": "^2.14.6",
+        "sillytavern-transformers": "2.14.6",
         "simple-git": "^3.19.1",
         "tiktoken": "^1.0.15",
         "vectra": "^0.2.2",

public/css/character-group-overlay.css

@@ -99,6 +99,6 @@
 }
 
 #bulk_tag_shadow_popup #bulk_tag_popup #dialogue_popup_controls .menu_button {
-    width: 100px;
+    width: unset;
     padding: 0.25em;
 }

public/index.html

@@ -4121,6 +4121,10 @@
                                 <input id="world_import_dialog" type="checkbox" />
                                 <small data-i18n="Lorebook Import Dialog">Lorebook Import Dialog</small>
                             </label>
+                            <label data-newbie-hidden class="checkbox_label" for="enable_auto_select_input" title="Enable auto-select of input text in some text fields when clicking/selecting them. Applies to popup input textboxes, and possible other custom input fields." data-i18n="[title]Enable auto-select of input text in some text fields when clicking/selecting them. Applies to popup input textboxes, and possible other custom input fields.">
+                                <input id="enable_auto_select_input" type="checkbox" />
+                                <small data-i18n="Auto-select Input Text">Auto-select Input Text</small>
+                            </label>
                             <label class="checkbox_label" for="restore_user_input" title="Restore unsaved user input on page refresh." data-i18n="[title]Restore unsaved user input on page refresh">
                                 <input id="restore_user_input" type="checkbox" />
                                 <small data-i18n="Restore User Input">Restore User Input</small>
@@ -5008,7 +5012,7 @@
                 <div class="popup-crop-wrap">
                     <img class="popup-crop-image" src="">
                 </div>
-                <textarea class="popup-input text_pole result-control" rows="1" data-result="1" data-result-event="submit"></textarea>
+                <textarea class="popup-input text_pole result-control auto-select" rows="1" data-result="1" data-result-event="submit"></textarea>
                 <div class="popup-inputs"></div>
                 <div class="popup-controls">
                     <div class="popup-button-ok menu_button result-control" data-result="1" data-i18n="Delete">Delete</div>

public/script.js

@@ -5426,9 +5426,11 @@ export function cleanUpMessage(getMessage, isImpersonate, isContinue, displayInc
         getMessage = fixMarkdown(getMessage, false);
     }
 
-    const nameToTrim2 = isImpersonate ? name1 : name2;
+    const nameToTrim2 = isImpersonate
+        ? (!power_user.allow_name1_display ? name1 : '')
+        : (!power_user.allow_name2_display ? name2 : '');
 
-    if (getMessage.startsWith(nameToTrim2 + ':')) {
+    if (nameToTrim2 && getMessage.startsWith(nameToTrim2 + ':')) {
         getMessage = getMessage.replace(nameToTrim2 + ':', '');
         getMessage = getMessage.trimStart();
     }
@@ -8390,6 +8392,9 @@ const CONNECT_API_MAP = {
     },
 };
 
+// Collect all unique API names in an array
+export const UNIQUE_APIS = [...new Set(Object.values(CONNECT_API_MAP).map(x => x.selected))];
+
 // Fill connections map from textgen_types and chat_completion_sources
 for (const textGenType of Object.values(textgen_types)) {
     if (CONNECT_API_MAP[textGenType]) continue;
@@ -8964,9 +8969,6 @@ jQuery(async function () {
         return '';
     }
 
-    // Collect all unique API names in an array
-    const uniqueAPIs = [...new Set(Object.values(CONNECT_API_MAP).map(x => x.selected))];
-
     SlashCommandParser.addCommandObject(SlashCommand.fromProps({
         name: 'dupe',
         callback: duplicateCharacter,
@@ -8975,13 +8977,13 @@ jQuery(async function () {
     SlashCommandParser.addCommandObject(SlashCommand.fromProps({
         name: 'api',
         callback: connectAPISlash,
+        returns: 'the current API',
         unnamedArgumentList: [
             SlashCommandArgument.fromProps({
                 description: 'API to connect to',
                 typeList: [ARGUMENT_TYPE.STRING],
-                isRequired: false,
                 enumList: Object.entries(CONNECT_API_MAP).map(([api, { selected }]) =>
-                    new SlashCommandEnumValue(api, selected, enumTypes.getBasedOnIndex(uniqueAPIs.findIndex(x => x === selected)),
+                    new SlashCommandEnumValue(api, selected, enumTypes.getBasedOnIndex(UNIQUE_APIS.findIndex(x => x === selected)),
                         selected[0].toUpperCase() ?? enumIcons.default)),
             }),
         ],
@@ -10655,6 +10657,15 @@ jQuery(async function () {
     $(document).on('click', '.open_alternate_greetings', openAlternateGreetings);
     /* $('#set_character_world').on('click', openCharacterWorldPopup); */
 
+    $(document).on('focus', 'input.auto-select, textarea.auto-select', function () {
+        if (!power_user.enable_auto_select_input) return;
+        const control = $(this)[0];
+        if (control instanceof HTMLInputElement || control instanceof HTMLTextAreaElement) {
+            control.select();
+            console.debug('Auto-selecting content of input control', control);
+        }
+    });
+
     $(document).keyup(function (e) {
         if (e.key === 'Escape') {
             const isEditVisible = $('#curEditTextarea').is(':visible');
@@ -10738,7 +10749,7 @@ jQuery(async function () {
                 }
             } break;
             case 'import_tags': {
-                await importTags(characters[this_chid], { forceShow: true });
+                await importTags(characters[this_chid], { importSetting: tag_import_setting.ASK });
             } break;
             /*case 'delete_button':
                 popup_type = "del_ch";

public/scripts/BulkEditOverlay.js

@@ -18,7 +18,7 @@ import {
 import { favsToHotswap } from './RossAscends-mods.js';
 import { hideLoader, showLoader } from './loader.js';
 import { convertCharacterToPersona } from './personas.js';
-import { createTagInput, getTagKeyForEntity, getTagsList, printTagList, tag_map, compareTagsForSort, removeTagFromMap } from './tags.js';
+import { createTagInput, getTagKeyForEntity, getTagsList, printTagList, tag_map, compareTagsForSort, removeTagFromMap, importTags, tag_import_setting } from './tags.js';
 
 /**
  * Static object representing the actions of the
@@ -197,10 +197,10 @@ class BulkTagPopupHandler {
     #getHtml = () => {
         const characterData = JSON.stringify({ characterIds: this.characterIds });
         return `<div id="bulk_tag_shadow_popup">
-            <div id="bulk_tag_popup">
+            <div id="bulk_tag_popup" class="wider_dialogue_popup">
                 <div id="bulk_tag_popup_holder">
                     <h3 class="marginBot5">Modify tags of ${this.characterIds.length} characters</h3>
-                    <small class="bulk_tags_desc m-b-1">Add or remove the mutual tags of all selected characters.</small>
+                    <small class="bulk_tags_desc m-b-1">Add or remove the mutual tags of all selected characters. Import all or existing tags for all selected characters.</small>
                     <div id="bulk_tags_avatars_block" class="avatars_inline avatars_inline_small tags tags_inline"></div>
                     <br>
                     <div id="bulk_tags_div" class="marginBot5" data-characters='${characterData}'>
@@ -219,6 +219,12 @@ class BulkTagPopupHandler {
                             <i class="fa-solid fa-trash-can margin-right-10px"></i>
                             Mutual
                         </div>
+                        <div id="bulk_tag_popup_import_all_tags" class="menu_button" title="Import all tags from selected characters" data-i18n="[title]Import all tags from selected characters">
+                            Import All
+                        </div>
+                        <div id="bulk_tag_popup_import_existing_tags" class="menu_button" title="Import existing tags from selected characters" data-i18n="[title]Import existing tags from selected characters">
+                            Import Existing
+                        </div>
                         <div id="bulk_tag_popup_cancel" class="menu_button" data-i18n="Cancel">Close</div>
                     </div>
                 </div>
@@ -254,6 +260,30 @@ class BulkTagPopupHandler {
         document.querySelector('#bulk_tag_popup_reset').addEventListener('click', this.resetTags.bind(this));
         document.querySelector('#bulk_tag_popup_remove_mutual').addEventListener('click', this.removeMutual.bind(this));
         document.querySelector('#bulk_tag_popup_cancel').addEventListener('click', this.hide.bind(this));
+        document.querySelector('#bulk_tag_popup_import_all_tags').addEventListener('click', this.importAllTags.bind(this));
+        document.querySelector('#bulk_tag_popup_import_existing_tags').addEventListener('click', this.importExistingTags.bind(this));
+    }
+
+    /**
+     * Import existing tags for all selected characters
+     */
+    async importExistingTags() {
+        for (const characterId of this.characterIds) {
+            await importTags(characters[characterId], { importSetting: tag_import_setting.ONLY_EXISTING });
+        }
+
+        $('#bulkTagList').empty();
+    }
+
+    /**
+     * Import all tags for all selected characters
+     */
+    async importAllTags() {
+        for (const characterId of this.characterIds) {
+            await importTags(characters[characterId], { importSetting: tag_import_setting.ALL });
+        }
+
+        $('#bulkTagList').empty();
     }
 
     /**

public/scripts/RossAscends-mods.js

@@ -954,6 +954,11 @@ export function initRossMods() {
      * @param {KeyboardEvent} event
      */
     async function processHotkeys(event) {
+        // Default hotkeys and shortcuts shouldn't work if any popup is currently open
+        if (Popup.util.isPopupOpen()) {
+            return;
+        }
+
         //Enter to send when send_textarea in focus
         if (document.activeElement == hotkeyTargets['send_textarea']) {
             const sendOnEnter = shouldSendOnEnter();
@@ -1107,10 +1112,6 @@ export function initRossMods() {
         }
 
         if (event.key == 'Escape') { //closes various panels
-            // Do not close panels if we are currently inside a popup
-            if (Popup.util.isPopupOpen())
-                return;
-
             //dont override Escape hotkey functions from script.js
             //"close edit box" and "cancel stream generation".
             if ($('#curEditTextarea').is(':visible') || $('#mes_stop').is(':visible')) {

public/scripts/extensions/caption/settings.html

@@ -38,6 +38,7 @@
                         <option data-type="openai" value="gpt-4-turbo">gpt-4-turbo</option>
                         <option data-type="openai" value="gpt-4o">gpt-4o</option>
                         <option data-type="openai" value="gpt-4o-mini">gpt-4o-mini</option>
+                        <option data-type="openai" value="chatgpt-4o-latest">chatgpt-4o-latest</option>
                         <option data-type="anthropic" value="claude-3-5-sonnet-20240620">claude-3-5-sonnet-20240620</option>
                         <option data-type="anthropic" value="claude-3-opus-20240229">claude-3-opus-20240229</option>
                         <option data-type="anthropic" value="claude-3-sonnet-20240229">claude-3-sonnet-20240229</option>

public/scripts/extensions/vectors/index.js

@@ -401,7 +401,7 @@ async function processFiles(chat) {
         const dataBankCollectionIds = await ingestDataBankAttachments();
 
         if (dataBankCollectionIds.length) {
-            const queryText = await getQueryText(chat);
+            const queryText = await getQueryText(chat, 'file');
             await injectDataBankChunks(queryText, dataBankCollectionIds);
         }
 
@@ -435,7 +435,7 @@ async function processFiles(chat) {
                 await vectorizeFile(fileText, fileName, collectionId, settings.chunk_size, settings.overlap_percent);
             }
 
-            const queryText = await getQueryText(chat);
+            const queryText = await getQueryText(chat, 'file');
             const fileChunks = await retrieveFileChunks(queryText, collectionId);
 
             message.mes = `${fileChunks}\n\n${message.mes}`;
@@ -596,7 +596,7 @@ async function rearrangeChat(chat) {
             return;
         }
 
-        const queryText = await getQueryText(chat);
+        const queryText = await getQueryText(chat, 'chat');
 
         if (queryText.length === 0) {
             console.debug('Vectors: No text to query');
@@ -683,15 +683,16 @@ const onChatEvent = debounce(async () => await moduleWorker.update(), debounce_t
 /**
  * Gets the text to query from the chat
  * @param {object[]} chat Chat messages
+ * @param {'file'|'chat'|'world-info'} initiator Initiator of the query
  * @returns {Promise<string>} Text to query
  */
-async function getQueryText(chat) {
+async function getQueryText(chat, initiator) {
     let queryText = '';
     let i = 0;
 
     let hashedMessages = chat.map(x => ({ text: String(substituteParams(x.mes)) }));
 
-    if (settings.summarize && settings.summarize_sent) {
+    if (initiator === 'chat' && settings.enabled_chats && settings.summarize && settings.summarize_sent) {
         hashedMessages = await summarize(hashedMessages, settings.summary_source);
     }
 
@@ -1279,7 +1280,7 @@ async function activateWorldInfo(chat) {
     }
 
     // Perform a multi-query
-    const queryText = await getQueryText(chat);
+    const queryText = await getQueryText(chat, 'world-info');
 
     if (queryText.length === 0) {
         console.debug('Vectors: No text to query for WI');

public/scripts/openai.js

@@ -4763,12 +4763,13 @@ export function isImageInliningSupported() {
         'gpt-4-turbo',
         'gpt-4o',
         'gpt-4o-mini',
+        'chatgpt-4o-latest',
         'yi-vision',
     ];
 
     switch (oai_settings.chat_completion_source) {
         case chat_completion_sources.OPENAI:
-            return visionSupportedModels.some(model => oai_settings.openai_model.includes(model) && !oai_settings.openai_model.includes('chatgpt-4o-latest') && !oai_settings.openai_model.includes('gpt-4-turbo-preview'));
+            return visionSupportedModels.some(model => oai_settings.openai_model.includes(model) && !oai_settings.openai_model.includes('gpt-4-turbo-preview'));
         case chat_completion_sources.MAKERSUITE:
             return visionSupportedModels.some(model => oai_settings.google_model.includes(model));
         case chat_completion_sources.CLAUDE:

public/scripts/popup.js

@@ -596,7 +596,7 @@ export class Popup {
 
         /** @returns {boolean} Checks if any modal popup dialog is open */
         isPopupOpen() {
-            return Popup.util.popups.length > 0;
+            return Popup.util.popups.filter(x => x.dlg.hasAttribute('open')).length > 0;
         },
 
         /**

public/scripts/power-user.js

@@ -202,6 +202,7 @@ let power_user = {
     trim_spaces: true,
     relaxed_api_urls: false,
     world_import_dialog: true,
+    enable_auto_select_input: false,
     tag_import_setting: tag_import_setting.ASK,
     disable_group_trimming: false,
     single_line: false,
@@ -1611,6 +1612,7 @@ async function loadPowerUserSettings(settings, data) {
     $('#single_line').prop('checked', power_user.single_line);
     $('#relaxed_api_urls').prop('checked', power_user.relaxed_api_urls);
     $('#world_import_dialog').prop('checked', power_user.world_import_dialog);
+    $('#enable_auto_select_input').prop('checked', power_user.enable_auto_select_input);
     $('#trim_spaces').prop('checked', power_user.trim_spaces);
     $('#continue_on_send').prop('checked', power_user.continue_on_send);
     $('#quick_continue').prop('checked', power_user.quick_continue);
@@ -3788,6 +3790,12 @@ $(document).ready(() => {
         saveSettingsDebounced();
     });
 
+    $('#enable_auto_select_input').on('input', function () {
+        const value = !!$(this).prop('checked');
+        power_user.enable_auto_select_input = value;
+        saveSettingsDebounced();
+    });
+
     $('#spoiler_free_mode').on('input', function () {
         power_user.spoiler_free_mode = !!$(this).prop('checked');
         switchSpoilerMode();

public/scripts/showdown-underscore.js

@@ -7,9 +7,19 @@ export const markdownUnderscoreExt = () => {
         }
 
         return [{
-            type: 'lang',
+            type: 'output',
-            regex: new RegExp('\\b(?<!_)_(?!_)(.*?)(?<!_)_(?!_)\\b', 'g'),
+            regex: new RegExp('(<code>[\\s\\S]*?<\\/code>)|(?<!\\S)_(?!_)([^_\\n]+?)(?<!_)_(?!\\w)', 'g'),
-            replace: '<em>$1</em>',
+            replace: function(match, codeContent, italicContent) {
+                if (codeContent) {
+                    // If it's inside <code> tags, return unchanged
+                    return match;
+                } else if (italicContent) {
+                    // If it's an italic group, apply the replacement
+                    return '<em>' + italicContent + '</em>';
+                }
+                // If none of the conditions are met, return the original match
+                return match;
+            },
         }];
     } catch (e) {
         console.error('Error in Showdown-underscore extension:', e);

public/scripts/slash-commands.js

@@ -1,7 +1,9 @@
 import {
     Generate,
+    UNIQUE_APIS,
     activateSendButtons,
     addOneMessage,
+    api_server,
     callPopup,
     characters,
     chat,
@@ -49,8 +51,8 @@ import { findGroupMemberId, groups, is_group_generating, openGroupById, resetSel
 import { chat_completion_sources, oai_settings, setupChatCompletionPromptManager } from './openai.js';
 import { autoSelectPersona, retriggerFirstMessageOnEmptyChat, setPersonaLockState, togglePersonaLock, user_avatar } from './personas.js';
 import { addEphemeralStoppingString, chat_styles, flushEphemeralStoppingStrings, power_user } from './power-user.js';
-import { textgen_types, textgenerationwebui_settings } from './textgen-settings.js';
+import { SERVER_INPUTS, textgen_types, textgenerationwebui_settings } from './textgen-settings.js';
-import { decodeTextTokens, getFriendlyTokenizerName, getTextTokens, getTokenCountAsync } from './tokenizers.js';
+import { decodeTextTokens, getAvailableTokenizers, getFriendlyTokenizerName, getTextTokens, getTokenCountAsync, selectTokenizer } from './tokenizers.js';
 import { debounce, delay, isFalseBoolean, isTrueBoolean, showFontAwesomePicker, stringToRange, trimToEndSentence, trimToStartSentence, waitUntilCondition } from './utils.js';
 import { registerVariableCommands, resolveVariable } from './variables.js';
 import { background_settings } from './backgrounds.js';
@@ -1496,7 +1498,8 @@ export function initDefaultSlashCommands() {
         ],
         helpString: 'Sets the specified prompt manager entry/entries on or off.',
     }));
-    SlashCommandParser.addCommandObject(SlashCommand.fromProps({ name: 'pick-icon',
+    SlashCommandParser.addCommandObject(SlashCommand.fromProps({
+        name: 'pick-icon',
         callback: async () => ((await showFontAwesomePicker()) ?? false).toString(),
         returns: 'The chosen icon name or false if cancelled.',
         helpString: `
@@ -1511,6 +1514,72 @@ export function initDefaultSlashCommands() {
                 </div>
             `,
     }));
+    SlashCommandParser.addCommandObject(SlashCommand.fromProps({
+        name: 'api-url',
+        callback: setApiUrlCallback,
+        returns: 'the current API url',
+        aliases: ['server'],
+        namedArgumentList: [
+            SlashCommandNamedArgument.fromProps({
+                name: 'api',
+                description: 'API to set/get the URL for - if not provided, current API is used',
+                typeList: [ARGUMENT_TYPE.STRING],
+                enumList: [
+                    new SlashCommandEnumValue('custom', 'custom OpenAI-compatible', enumTypes.getBasedOnIndex(UNIQUE_APIS.findIndex(x => x === 'openai')), 'O'),
+                    new SlashCommandEnumValue('kobold', 'KoboldAI Classic', enumTypes.getBasedOnIndex(UNIQUE_APIS.findIndex(x => x === 'kobold')), 'K'),
+                    ...Object.values(textgen_types).map(api => new SlashCommandEnumValue(api, null, enumTypes.getBasedOnIndex(UNIQUE_APIS.findIndex(x => x === 'textgenerationwebui')), 'T')),
+                ],
+            }),
+            SlashCommandNamedArgument.fromProps({
+                name: 'connect',
+                description: 'Whether to auto-connect to the API after setting the URL',
+                typeList: [ARGUMENT_TYPE.BOOLEAN],
+                defaultValue: 'true',
+                enumList: commonEnumProviders.boolean('trueFalse')(),
+            }),
+        ],
+        unnamedArgumentList: [
+            SlashCommandArgument.fromProps({
+                description: 'API url to connect to',
+                typeList: [ARGUMENT_TYPE.STRING],
+            }),
+        ],
+        helpString: `
+            <div>
+                Set the API url / server url for the currently selected API, including the port. If no argument is provided, it will return the current API url.
+            </div>
+            <div>
+                If a manual API is provided to <b>set</b> the URL, make sure to set <code>connect=false</code>, as auto-connect only works for the currently selected API,
+                or consider switching to it with <code>/api</code> first.
+            </div>
+            <div>
+                This slash command works for most of the Text Completion sources, KoboldAI Classic, and also Custom OpenAI compatible for the Chat Completion sources. If unsure which APIs are supported,
+                check the auto-completion of the optional <code>api</code> argument of this command.
+            </div>
+        `,
+    }));
+    SlashCommandParser.addCommandObject(SlashCommand.fromProps({
+        name: 'tokenizer',
+        callback: selectTokenizerCallback,
+        returns: 'current tokenizer',
+        unnamedArgumentList: [
+            SlashCommandArgument.fromProps({
+                description: 'tokenizer name',
+                typeList: [ARGUMENT_TYPE.STRING],
+                enumList: getAvailableTokenizers().map(tokenizer =>
+                    new SlashCommandEnumValue(tokenizer.tokenizerKey, tokenizer.tokenizerName, enumTypes.enum, enumIcons.default)),
+            }),
+        ],
+        helpString: `
+            <div>
+                Selects tokenizer by name. Gets the current tokenizer if no name is provided.
+            </div>
+            <div>
+                <strong>Available tokenizers:</strong>
+                <pre><code>${getAvailableTokenizers().map(t => t.tokenizerKey).join(', ')}</code></pre>
+            </div>
+        `,
+    }));
 
     registerVariableCommands();
 }
@@ -1788,7 +1857,7 @@ async function popupCallback(args, value) {
     return String(value);
 }
 
-function getMessagesCallback(args, value) {
+async function getMessagesCallback(args, value) {
     const includeNames = !isFalseBoolean(args?.names);
     const includeHidden = isTrueBoolean(args?.hidden);
     const role = args?.role;
@@ -1821,33 +1890,34 @@ function getMessagesCallback(args, value) {
         throw new Error(`Invalid role provided. Expected one of: system, assistant, user. Got: ${role}`);
     };
 
-    const messages = [];
+    const processMessage = async (mesId) => {
-
+        const msg = chat[mesId];
-    for (let messageId = range.start; messageId <= range.end; messageId++) {
+        if (!msg) {
-        const message = chat[messageId];
+            console.warn(`WARN: No message found with ID ${mesId}`);
-        if (!message) {
+            return null;
-            console.warn(`WARN: No message found with ID ${messageId}`);
-            continue;
         }
 
-        if (role && !filterByRole(message)) {
+        if (role && !filterByRole(msg)) {
-            console.debug(`/messages: Skipping message with ID ${messageId} due to role filter`);
+            console.debug(`/messages: Skipping message with ID ${mesId} due to role filter`);
-            continue;
+            return null;
         }
 
-        if (!includeHidden && message.is_system) {
+        if (!includeHidden && msg.is_system) {
-            console.debug(`/messages: Skipping hidden message with ID ${messageId}`);
+            console.debug(`/messages: Skipping hidden message with ID ${mesId}`);
-            continue;
+            return null;
         }
 
-        if (includeNames) {
+        return includeNames ? `${msg.name}: ${msg.mes}` : msg.mes;
-            messages.push(`${message.name}: ${message.mes}`);
+    };
-        } else {
-            messages.push(message.mes);
-        }
-    }
 
-    return messages.join('\n\n');
+    const messagePromises = [];
+
+    for (let rInd = range.start; rInd <= range.end; ++rInd)
+        messagePromises.push(processMessage(rInd));
+
+    const messages = await Promise.all(messagePromises);
+
+    return messages.filter(m => m !== null).join('\n\n');
 }
 
 async function runCallback(args, name) {
@@ -3418,6 +3488,123 @@ function setPromptEntryCallback(args, targetState) {
     return '';
 }
 
+/**
+ * Sets the API URL and triggers the text generation web UI button click.
+ *
+ * @param {object} args - named args
+ * @param {string?} [args.api=null] - the API name to set/get the URL for
+ * @param {string?} [args.connect=true] - whether to connect to the API after setting
+ * @param {string} url - the API URL to set
+ * @returns {Promise<string>}
+ */
+async function setApiUrlCallback({ api = null, connect = 'true' }, url) {
+    const autoConnect = isTrueBoolean(connect);
+
+    // Special handling for Chat Completion Custom OpenAI compatible, that one can also support API url handling
+    const isCurrentlyCustomOpenai = main_api === 'openai' && oai_settings.chat_completion_source === chat_completion_sources.CUSTOM;
+    if (api === chat_completion_sources.CUSTOM || (!api && isCurrentlyCustomOpenai)) {
+        if (!url) {
+            return oai_settings.custom_url ?? '';
+        }
+
+        if (!isCurrentlyCustomOpenai && autoConnect) {
+            toastr.warning('Custom OpenAI API is not the currently selected API, so we cannot do an auto-connect. Consider switching to it via /api beforehand.');
+            return '';
+        }
+
+        $('#custom_api_url_text').val(url).trigger('input');
+
+        if (autoConnect) {
+            $('#api_button_openai').trigger('click');
+        }
+
+        return url;
+    }
+
+    // Special handling for Kobold Classic API
+    const isCurrentlyKoboldClassic = main_api === 'kobold';
+    if (api === 'kobold' || (!api && isCurrentlyKoboldClassic)) {
+        if (!url) {
+            return api_server ?? '';
+        }
+
+        if (!isCurrentlyKoboldClassic && autoConnect) {
+            toastr.warning('Kobold Classic API is not the currently selected API, so we cannot do an auto-connect. Consider switching to it via /api beforehand.');
+            return '';
+        }
+
+        $('#api_url_text').val(url).trigger('input');
+        // trigger blur debounced, so we hide the autocomplete menu
+        setTimeout(() => $('#api_url_text').trigger('blur'), 1);
+
+        if (autoConnect) {
+            $('#api_button').trigger('click');
+        }
+
+        return api_server ?? '';
+    }
+
+    // Do some checks and get the api type we are targeting with this command
+    if (api && !Object.values(textgen_types).includes(api)) {
+        toastr.warning(`API '${api}' is not a valid text_gen API.`);
+        return '';
+    }
+    if (!api && !Object.values(textgen_types).includes(textgenerationwebui_settings.type)) {
+        toastr.warning(`API '${textgenerationwebui_settings.type}' is not a valid text_gen API.`);
+        return '';
+    }
+    if (api && url && autoConnect && api !== textgenerationwebui_settings.type) {
+        toastr.warning(`API '${api}' is not the currently selected API, so we cannot do an auto-connect. Consider switching to it via /api beforehand.`);
+        return '';
+    }
+    const type = api || textgenerationwebui_settings.type;
+
+    const inputSelector = SERVER_INPUTS[type];
+    if (!inputSelector) {
+        toastr.warning(`API '${type}' does not have a server url input.`);
+        return '';
+    }
+
+    // If no url was provided, return the current one
+    if (!url) {
+        return textgenerationwebui_settings.server_urls[type] ?? '';
+    }
+
+    // else, we want to actually set the url
+    $(inputSelector).val(url).trigger('input');
+    // trigger blur debounced, so we hide the autocomplete menu
+    setTimeout(() => $(inputSelector).trigger('blur'), 1);
+
+    // Trigger the auto connect via connect button, if requested
+    if (autoConnect) {
+        $('#api_button_textgenerationwebui').trigger('click');
+    }
+
+    // We still re-acquire the value, as it might have been modified by the validation on connect
+    return textgenerationwebui_settings.server_urls[type] ?? '';
+}
+
+async function selectTokenizerCallback(_, name) {
+    if (!name) {
+        return getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === power_user.tokenizer)?.tokenizerKey ?? '';
+    }
+
+    const tokenizers = getAvailableTokenizers();
+    const fuse = new Fuse(tokenizers, { keys: ['tokenizerKey', 'tokenizerName'] });
+    const result = fuse.search(name);
+
+    if (result.length === 0) {
+        toastr.warning(`Tokenizer "${name}" not found`);
+        return '';
+    }
+
+    /** @type {import('./tokenizers.js').Tokenizer} */
+    const foundTokenizer = result[0].item;
+    selectTokenizer(foundTokenizer.tokenizerId);
+
+    return foundTokenizer.tokenizerKey;
+}
+
 export let isExecutingCommandsFromChatInput = false;
 export let commandsFromChatInputAbortController;
 

public/scripts/tags.js

@@ -708,12 +708,12 @@ const ANTI_TROLL_MAX_TAGS = 15;
  *
  * @param {Character} character - The character
  * @param {object} [options] - Options
- * @param {boolean} [options.forceShow=false] - Whether to force showing the import dialog
+ * @param {tag_import_setting} [options.importSetting=null] - Force a tag import setting
  * @returns {Promise<boolean>} Boolean indicating whether any tag was imported
  */
-async function importTags(character, { forceShow = false } = {}) {
+async function importTags(character, { importSetting = null } = {}) {
     // Gather the tags to import based on the selected setting
-    const tagNamesToImport = await handleTagImport(character, { forceShow });
+    const tagNamesToImport = await handleTagImport(character, { importSetting });
     if (!tagNamesToImport?.length) {
         console.debug('No tags to import');
         return;
@@ -732,10 +732,10 @@ async function importTags(character, { forceShow = false } = {}) {
  *
  * @param {Character} character - The character
  * @param {object} [options] - Options
- * @param {boolean} [options.forceShow=false] - Whether to force showing the import dialog
+ * @param {tag_import_setting} [options.importSetting=null] - Force a tag import setting
  * @returns {Promise<string[]>} Array of strings representing the tags to import
  */
-async function handleTagImport(character, { forceShow = false } = {}) {
+async function handleTagImport(character, { importSetting = null } = {}) {
     /** @type {string[]} */
     const importTags = character.tags.map(t => t.trim()).filter(t => t)
         .filter(t => !IMPORT_EXLCUDED_TAGS.includes(t))
@@ -745,9 +745,9 @@ async function handleTagImport(character, { forceShow = false } = {}) {
         .map(newTag);
     const folderTags = getOpenBogusFolders();
 
-    // Choose the setting for this dialog. If from settings, verify the setting really exists, otherwise take "ASK".
+    // Choose the setting for this dialog. First check override, then saved setting or finally use "ASK".
-    const setting = forceShow ? tag_import_setting.ASK
+    const setting = importSetting ? importSetting :
-        : Object.values(tag_import_setting).find(setting => setting === power_user.tag_import_setting) ?? tag_import_setting.ASK;
+        Object.values(tag_import_setting).find(setting => setting === power_user.tag_import_setting) ?? tag_import_setting.ASK;
 
     switch (setting) {
         case tag_import_setting.ALL:

public/scripts/textgen-settings.js

@@ -95,7 +95,7 @@ let DREAMGEN_SERVER = 'https://dreamgen.com';
 let OPENROUTER_SERVER = 'https://openrouter.ai/api';
 let FEATHERLESS_SERVER = 'https://api.featherless.ai/v1';
 
-const SERVER_INPUTS = {
+export const SERVER_INPUTS = {
     [textgen_types.OOBA]: '#textgenerationwebui_api_url_text',
     [textgen_types.VLLM]: '#vllm_api_url_text',
     [textgen_types.APHRODITE]: '#aphrodite_api_url_text',
@@ -1071,6 +1071,34 @@ function getLogprobsNumber() {
     return 10;
 }
 
+/**
+ * Replaces {{macro}} in a comma-separated or serialized JSON array string.
+ * @param {string} str Input string
+ * @returns {string} Output string
+ */
+function replaceMacrosInList(str) {
+    if (!str || typeof str !== 'string') {
+        return str;
+    }
+
+    try {
+        const array = JSON.parse(str);
+        if (!Array.isArray(array)) {
+            throw new Error('Not an array');
+        }
+        for (let i = 0; i < array.length; i++) {
+            array[i] = substituteParams(array[i]);
+        }
+        return JSON.stringify(array);
+    } catch {
+        const array = str.split(',');
+        for (let i = 0; i < array.length; i++) {
+            array[i] = substituteParams(array[i]);
+        }
+        return array.join(',');
+    }
+}
+
 export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate, isContinue, cfgValues, type) {
     const canMultiSwipe = !isContinue && !isImpersonate && type !== 'quiet';
     const dynatemp = isDynamicTemperatureSupported();
@@ -1110,7 +1138,7 @@ export function getTextGenGenerationData(finalPrompt, maxTokens, isImpersonate,
         'dry_allowed_length': settings.dry_allowed_length,
         'dry_multiplier': settings.dry_multiplier,
         'dry_base': settings.dry_base,
-        'dry_sequence_breakers': settings.dry_sequence_breakers,
+        'dry_sequence_breakers': replaceMacrosInList(settings.dry_sequence_breakers),
         'dry_penalty_last_n': settings.dry_penalty_last_n,
         'max_tokens_second': settings.max_tokens_second,
         'sampler_priority': settings.type === OOBA ? settings.sampler_priority : undefined,

public/scripts/tokenizers.js

@@ -147,10 +147,46 @@ async function resetTokenCache() {
     }
 }
 
+/**
+ * @typedef {object} Tokenizer
+ * @property {number} tokenizerId - The id of the tokenizer option
+ * @property {string} tokenizerKey - Internal name/key of the tokenizer
+ * @property {string} tokenizerName - Human-readable detailed name of the tokenizer (as displayed in the UI)
+ */
+
+/**
+ * Gets all tokenizers available to the user.
+ * @returns {Tokenizer[]} Tokenizer info.
+ */
+export function getAvailableTokenizers() {
+    const tokenizerOptions = $('#tokenizer').find('option').toArray();
+    return tokenizerOptions.map(tokenizerOption => ({
+        tokenizerId: Number(tokenizerOption.value),
+        tokenizerKey: Object.entries(tokenizers).find(([_, value]) => value === Number(tokenizerOption.value))[0].toLocaleLowerCase(),
+        tokenizerName: tokenizerOption.text,
+    }))
+}
+
+/**
+ * Selects tokenizer if not already selected.
+ * @param {number} tokenizerId Tokenizer ID.
+ */
+export function selectTokenizer(tokenizerId) {
+    if (tokenizerId !== power_user.tokenizer) {
+        const tokenizer = getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === tokenizerId);
+        if (!tokenizer) {
+            console.warn('Failed to find tokenizer with id', tokenizerId);
+            return;
+        }
+        $('#tokenizer').val(tokenizer.tokenizerId).trigger('change');
+        toastr.info(`Tokenizer: "${tokenizer.tokenizerName}" selected`);
+    }
+}
+
 /**
  * Gets the friendly name of the current tokenizer.
  * @param {string} forApi API to get the tokenizer for. Defaults to the main API.
- * @returns { { tokenizerName: string, tokenizerId: number } } Tokenizer info
+ * @returns {Tokenizer} Tokenizer info
  */
 export function getFriendlyTokenizerName(forApi) {
     if (!forApi) {
@@ -185,7 +221,9 @@ export function getFriendlyTokenizerName(forApi) {
         ? tokenizers.OPENAI
         : tokenizerId;
 
-    return { tokenizerName, tokenizerId };
+    const tokenizerKey = Object.entries(tokenizers).find(([_, value]) => value === tokenizerId)[0].toLocaleLowerCase();
+
+    return { tokenizerName, tokenizerKey, tokenizerId };
 }
 
 /**

server.js

@@ -787,17 +787,17 @@ function logSecurityAlert(message) {
  */
 function handleServerListenFail(v6Failed, v4Failed) {
     if (v6Failed && !enableIPv4) {
-        console.error('fatal error: Failed to start server on IPv6 and IPv4 disabled');
+        console.error(color.red('fatal error: Failed to start server on IPv6 and IPv4 disabled'));
         process.exit(1);
     }
 
     if (v4Failed && !enableIPv6) {
-        console.error('fatal error: Failed to start server on IPv4 and IPv6 disabled');
+        console.error(color.red('fatal error: Failed to start server on IPv4 and IPv6 disabled'));
         process.exit(1);
     }
 
     if (v6Failed && v4Failed) {
-        console.error('fatal error: Failed to start server on both IPv6 and IPv4');
+        console.error(color.red('fatal error: Failed to start server on both IPv6 and IPv4'));
         process.exit(1);
     }
 }
@@ -846,9 +846,8 @@ async function startHTTPorHTTPS() {
         try {
             await createFunc(tavernUrlV6);
         } catch (error) {
-            if (enableIPv4) {
+            console.error('non-fatal error: failed to start server on IPv6');
-                console.error('non-fatal error: failed to start server on IPv6', error);
+            console.error(error);
-            }
 
             v6Failed = true;
         }
@@ -858,9 +857,8 @@ async function startHTTPorHTTPS() {
         try {
             await createFunc(tavernUrl);
         } catch (error) {
-            if (enableIPv6) {
+            console.error('non-fatal error: failed to start server on IPv4');
-                console.error('non-fatal error: failed to start server on IPv4', error);
+            console.error(error);
-            }
 
             v4Failed = true;
         }

src/endpoints/translate.js

@@ -1,6 +1,7 @@
 const fetch = require('node-fetch').default;
 const https = require('https');
 const express = require('express');
+const iconv = require('iconv-lite');
 const { readSecret, SECRET_KEYS } = require('./secrets');
 const { getConfigValue, uuidv4 } = require('../util');
 const { jsonParser } = require('../express-common');
@@ -80,16 +81,18 @@ router.post('/google', jsonParser, async (request, response) => {
         const url = generateRequestUrl(text, { to: lang });
 
         https.get(url, (resp) => {
-            let data = '';
+            const data = [];
 
             resp.on('data', (chunk) => {
-                data += chunk;
+                data.push(chunk);
             });
 
             resp.on('end', () => {
                 try {
-                    const result = normaliseResponse(JSON.parse(data));
+                    const decodedData = iconv.decode(Buffer.concat(data), 'utf-8');
+                    const result = normaliseResponse(JSON.parse(decodedData));
                     console.log('Translated text: ' + result.text);
+                    response.setHeader('Content-Type', 'text/plain; charset=utf-8');
                     return response.send(result.text);
                 } catch (error) {
                     console.log('Translation error', error);

src/transformers.mjs

@@ -1,6 +1,7 @@
 import { pipeline, env, RawImage, Pipeline } from 'sillytavern-transformers';
 import { getConfigValue } from './util.js';
 import path from 'path';
+import fs from 'fs';
 
 configureTransformers();
 
@@ -48,7 +49,7 @@ const tasks = {
         configField: 'extras.textToSpeechModel',
         quantized: false,
     },
-}
+};
 
 /**
  * Gets a RawImage object from a base64-encoded image.
@@ -85,6 +86,36 @@ function getModelForTask(task) {
     }
 }
 
+async function migrateCacheToDataDir() {
+    const oldCacheDir = path.join(process.cwd(), 'cache');
+    const newCacheDir = path.join(global.DATA_ROOT, '_cache');
+
+    if (!fs.existsSync(newCacheDir)) {
+        fs.mkdirSync(newCacheDir, { recursive: true });
+    }
+
+    if (fs.existsSync(oldCacheDir) && fs.statSync(oldCacheDir).isDirectory()) {
+        const files = fs.readdirSync(oldCacheDir);
+
+        if (files.length === 0) {
+            return;
+        }
+
+        console.log('Migrating model cache files to data directory. Please wait...');
+
+        for (const file of files) {
+            try {
+                const oldPath = path.join(oldCacheDir, file);
+                const newPath = path.join(newCacheDir, file);
+                fs.cpSync(oldPath, newPath, { recursive: true, force: true });
+                fs.rmSync(oldPath, { recursive: true, force: true });
+            } catch (error) {
+                console.warn('Failed to migrate cache file. The model will be re-downloaded.', error);
+            }
+        }
+    }
+}
+
 /**
  * Gets the transformers.js pipeline for a given task.
  * @param {import('sillytavern-transformers').PipelineType} task The task to get the pipeline for
@@ -92,6 +123,8 @@ function getModelForTask(task) {
  * @returns {Promise<Pipeline>} Pipeline for the task
  */
 async function getPipeline(task, forceModel = '') {
+    await migrateCacheToDataDir();
+
     if (tasks[task].pipeline) {
         if (forceModel === '' || tasks[task].currentModel === forceModel) {
             return tasks[task].pipeline;
@@ -100,11 +133,11 @@ async function getPipeline(task, forceModel = '') {
         await tasks[task].pipeline.dispose();
     }
 
-    const cache_dir = path.join(process.cwd(), 'cache');
+    const cacheDir = path.join(global.DATA_ROOT, '_cache');
     const model = forceModel || getModelForTask(task);
     const localOnly = getConfigValue('extras.disableAutoDownload', false);
     console.log('Initializing transformers.js pipeline for task', task, 'with model', model);
-    const instance = await pipeline(task, model, { cache_dir, quantized: tasks[task].quantized ?? true, local_files_only: localOnly });
+    const instance = await pipeline(task, model, { cache_dir: cacheDir, quantized: tasks[task].quantized ?? true, local_files_only: localOnly });
     tasks[task].pipeline = instance;
     tasks[task].currentModel = model;
     return instance;

src/users.js

@@ -19,12 +19,6 @@ const AVATAR_PREFIX = 'avatar:';
 const ENABLE_ACCOUNTS = getConfigValue('enableUserAccounts', false);
 const ANON_CSRF_SECRET = crypto.randomBytes(64).toString('base64');
 
-/**
- * The root directory for user data.
- * @type {string}
- */
-let DATA_ROOT = './data';
-
 /**
  * Cache for user directories.
  * @type {Map<string, UserDirectoryList>}
@@ -138,7 +132,7 @@ async function migrateUserData() {
 
     console.log();
     console.log(color.magenta('Preparing to migrate user data...'));
-    console.log(`All public data will be moved to the ${DATA_ROOT} directory.`);
+    console.log(`All public data will be moved to the ${global.DATA_ROOT} directory.`);
     console.log('This process may take a while depending on the amount of data to move.');
     console.log(`Backups will be placed in the ${PUBLIC_DIRECTORIES.backups} directory.`);
     console.log(`The process will start in ${TIMEOUT} seconds. Press Ctrl+C to cancel.`);
@@ -352,11 +346,11 @@ function toAvatarKey(handle) {
  * @returns {Promise<void>}
  */
 async function initUserStorage(dataRoot) {
-    DATA_ROOT = dataRoot;
+    global.DATA_ROOT = dataRoot;
-    console.log('Using data root:', color.green(DATA_ROOT));
+    console.log('Using data root:', color.green(global.DATA_ROOT));
     console.log();
     await storage.init({
-        dir: path.join(DATA_ROOT, '_storage'),
+        dir: path.join(global.DATA_ROOT, '_storage'),
         ttl: false, // Never expire
     });
 
@@ -457,7 +451,7 @@ function getUserDirectories(handle) {
 
     const directories = structuredClone(USER_DIRECTORY_TEMPLATE);
     for (const key in directories) {
-        directories[key] = path.join(DATA_ROOT, handle, USER_DIRECTORY_TEMPLATE[key]);
+        directories[key] = path.join(global.DATA_ROOT, handle, USER_DIRECTORY_TEMPLATE[key]);
     }
     DIRECTORIES_CACHE.set(handle, directories);
     return directories;