Merge branch 'staging' into llm-expressions

2025-06-05 21:59:27 +02:00 · 2024-04-14 19:43:34 +03:00
parent fd0c16bf12 9cb777a4e9
commit bd6fe19bf1
34 changed files with 1084 additions and 411 deletions
--- a/public/scripts/BulkEditOverlay.js
+++ b/public/scripts/BulkEditOverlay.js
@@ -266,7 +266,7 @@ class BulkTagPopupHandler {
        printTagList($('#bulkTagList'), { tags: () => this.getMutualTags(), tagOptions: { removable: true } });

        // Tag input with resolvable list for the mutual tags to get redrawn, so that newly added tags get sorted correctly
-        createTagInput('#bulkTagInput', '#bulkTagList', { tags: () => this.getMutualTags(), tagOptions: { removable: true }});
+        createTagInput('#bulkTagInput', '#bulkTagList', { tags: () => this.getMutualTags(), tagOptions: { removable: true } });

        document.querySelector('#bulk_tag_popup_reset').addEventListener('click', this.resetTags.bind(this));
        document.querySelector('#bulk_tag_popup_remove_mutual').addEventListener('click', this.removeMutual.bind(this));
@@ -291,7 +291,7 @@ class BulkTagPopupHandler {
        // Find mutual tags for multiple characters
        const allTags = this.characterIds.map(cid => getTagsList(getTagKeyForEntity(cid)));
        const mutualTags = allTags.reduce((mutual, characterTags) =>
-            mutual.filter(tag => characterTags.some(cTag => cTag.id === tag.id))
+            mutual.filter(tag => characterTags.some(cTag => cTag.id === tag.id)),
        );

        this.currentMutualTags = mutualTags.sort(compareTagsForSort);
@@ -587,7 +587,7 @@ class BulkEditOverlay {
            this.container.removeEventListener('mouseup', cancelHold);
            this.container.removeEventListener('touchend', cancelHold);
        },
-            BulkEditOverlay.longPressDelay);
+        BulkEditOverlay.longPressDelay);
    };

    handleLongPressEnd = (event) => {
@@ -694,7 +694,7 @@ class BulkEditOverlay {
        } else {
            character.classList.remove(BulkEditOverlay.selectedClass);
            if (legacyBulkEditCheckbox) legacyBulkEditCheckbox.checked = false;
-            this.#selectedCharacters = this.#selectedCharacters.filter(item => String(characterId) !== item)
+            this.#selectedCharacters = this.#selectedCharacters.filter(item => String(characterId) !== item);
        }

        this.updateSelectedCount();
@@ -816,7 +816,7 @@ class BulkEditOverlay {
                    <span>Also delete the chat files</span>
                </label>
            </div>`;
-    }
+    };

    /**
     * Request user input before concurrently handle deletion
--- a/public/scripts/RossAscends-mods.js
+++ b/public/scripts/RossAscends-mods.js
@@ -34,7 +34,7 @@ import {
 } from './secrets.js';
 import { debounce, delay, getStringHash, isValidUrl } from './utils.js';
 import { chat_completion_sources, oai_settings } from './openai.js';
-import { getTokenCount } from './tokenizers.js';
+import { getTokenCountAsync } from './tokenizers.js';
 import { textgen_types, textgenerationwebui_settings as textgen_settings, getTextGenServer } from './textgen-settings.js';

 import Bowser from '../lib/bowser.min.js';
@@ -51,6 +51,7 @@ var SelectedCharacterTab = document.getElementById('rm_button_selected_ch');

 var connection_made = false;
 var retry_delay = 500;
+let counterNonce = Date.now();

 const observerConfig = { childList: true, subtree: true };
 const countTokensDebounced = debounce(RA_CountCharTokens, 1000);
@@ -202,24 +203,32 @@ $('#rm_ch_create_block').on('input', function () { countTokensDebounced(); });
 //when any input is made to the advanced editing popup textareas
 $('#character_popup').on('input', function () { countTokensDebounced(); });
 //function:
-export function RA_CountCharTokens() {
+export async function RA_CountCharTokens() {
+    counterNonce = Date.now();
+    const counterNonceLocal = counterNonce;
    let total_tokens = 0;
    let permanent_tokens = 0;

-    $('[data-token-counter]').each(function () {
-        const counter = $(this);
+    const tokenCounters = document.querySelectorAll('[data-token-counter]');
+    for (const tokenCounter of tokenCounters) {
+        if (counterNonceLocal !== counterNonce) {
+            return;
+        }
+
+        const counter = $(tokenCounter);
        const input = $(document.getElementById(counter.data('token-counter')));
        const isPermanent = counter.data('token-permanent') === true;
        const value = String(input.val());

        if (input.length === 0) {
            counter.text('Invalid input reference');
-            return;
+            continue;
        }

        if (!value) {
+            input.data('last-value-hash', '');
            counter.text(0);
-            return;
+            continue;
        }

        const valueHash = getStringHash(value);
@@ -230,13 +239,18 @@ export function RA_CountCharTokens() {
        } else {
            // We substitute macro for existing characters, but not for the character being created
            const valueToCount = menu_type === 'create' ? value : substituteParams(value);
-            const tokens = getTokenCount(valueToCount);
+            const tokens = await getTokenCountAsync(valueToCount);
+
+            if (counterNonceLocal !== counterNonce) {
+                return;
+            }
+
            counter.text(tokens);
            total_tokens += tokens;
            permanent_tokens += isPermanent ? tokens : 0;
            input.data('last-value-hash', valueHash);
        }
-    });
+    }

    // Warn if total tokens exceeds the limit of half the max context
    const tokenLimit = Math.max(((main_api !== 'openai' ? max_context : oai_settings.openai_max_context) / 2), 1024);
@@ -263,7 +277,7 @@ async function RA_autoloadchat() {
                await selectCharacterById(String(active_character_id));

                // Do a little tomfoolery to spoof the tag selector
-                const selectedCharElement = $(`#rm_print_characters_block .character_select[chid="${active_character_id}"]`)
+                const selectedCharElement = $(`#rm_print_characters_block .character_select[chid="${active_character_id}"]`);
                applyTagsOnCharacterSelect.call(selectedCharElement);
            }
        }
--- a/public/scripts/authors-note.js
+++ b/public/scripts/authors-note.js
@@ -11,7 +11,7 @@ import { selected_group } from './group-chats.js';
 import { extension_settings, getContext, saveMetadataDebounced } from './extensions.js';
 import { registerSlashCommand } from './slash-commands.js';
 import { getCharaFilename, debounce, delay } from './utils.js';
-import { getTokenCount } from './tokenizers.js';
+import { getTokenCountAsync } from './tokenizers.js';
 export { MODULE_NAME as NOTE_MODULE_NAME };

 const MODULE_NAME = '2_floating_prompt'; // <= Deliberate, for sorting lower than memory
@@ -84,9 +84,9 @@ function updateSettings() {
    setFloatingPrompt();
 }

-const setMainPromptTokenCounterDebounced = debounce((value) => $('#extension_floating_prompt_token_counter').text(getTokenCount(value)), 1000);
-const setCharaPromptTokenCounterDebounced = debounce((value) => $('#extension_floating_chara_token_counter').text(getTokenCount(value)), 1000);
-const setDefaultPromptTokenCounterDebounced = debounce((value) => $('#extension_floating_default_token_counter').text(getTokenCount(value)), 1000);
+const setMainPromptTokenCounterDebounced = debounce(async (value) => $('#extension_floating_prompt_token_counter').text(await getTokenCountAsync(value)), 1000);
+const setCharaPromptTokenCounterDebounced = debounce(async (value) => $('#extension_floating_chara_token_counter').text(await getTokenCountAsync(value)), 1000);
+const setDefaultPromptTokenCounterDebounced = debounce(async (value) => $('#extension_floating_default_token_counter').text(await getTokenCountAsync(value)), 1000);

 async function onExtensionFloatingPromptInput() {
    chat_metadata[metadata_keys.prompt] = $(this).val();
@@ -394,7 +394,7 @@ function onANMenuItemClick() {
    }
 }

-function onChatChanged() {
+async function onChatChanged() {
    loadSettings();
    setFloatingPrompt();
    const context = getContext();
@@ -402,7 +402,7 @@ function onChatChanged() {
    // Disable the chara note if in a group
    $('#extension_floating_chara').prop('disabled', context.groupId ? true : false);

-    const tokenCounter1 = chat_metadata[metadata_keys.prompt] ? getTokenCount(chat_metadata[metadata_keys.prompt]) : 0;
+    const tokenCounter1 = chat_metadata[metadata_keys.prompt] ? await getTokenCountAsync(chat_metadata[metadata_keys.prompt]) : 0;
    $('#extension_floating_prompt_token_counter').text(tokenCounter1);

    let tokenCounter2;
@@ -410,15 +410,13 @@ function onChatChanged() {
        const charaNote = extension_settings.note.chara.find((e) => e.name === getCharaFilename());

        if (charaNote) {
-            tokenCounter2 = getTokenCount(charaNote.prompt);
+            tokenCounter2 = await getTokenCountAsync(charaNote.prompt);
        }
    }

-    if (tokenCounter2) {
-        $('#extension_floating_chara_token_counter').text(tokenCounter2);
-    }
+    $('#extension_floating_chara_token_counter').text(tokenCounter2 || 0);

-    const tokenCounter3 = extension_settings.note.default ? getTokenCount(extension_settings.note.default) : 0;
+    const tokenCounter3 = extension_settings.note.default ? await getTokenCountAsync(extension_settings.note.default) : 0;
    $('#extension_floating_default_token_counter').text(tokenCounter3);
 }

--- a/public/scripts/chats.js
+++ b/public/scripts/chats.js
@@ -44,22 +44,29 @@ function isConvertible(type) {
 }

 /**
- * Mark message as hidden (system message).
- * @param {number} messageId Message ID
- * @param {JQuery<Element>} messageBlock Message UI element
- * @returns
+ * Mark a range of messages as hidden ("is_system") or not.
+ * @param {number} start Starting message ID
+ * @param {number} end Ending message ID (inclusive)
+ * @param {boolean} unhide If true, unhide the messages instead.
+ * @returns {Promise<void>}
 */
-export async function hideChatMessage(messageId, messageBlock) {
-    const chatId = getCurrentChatId();
+export async function hideChatMessageRange(start, end, unhide) {
+    if (!getCurrentChatId()) return;

-    if (!chatId || isNaN(messageId)) return;
+    if (isNaN(start)) return;
+    if (!end) end = start;
+    const hide = !unhide;

-    const message = chat[messageId];
+    for (let messageId = start; messageId <= end; messageId++) {
+        const message = chat[messageId];
+        if (!message) continue;

-    if (!message) return;
+        const messageBlock = $(`.mes[mesid="${messageId}"]`);
+        if (!messageBlock.length) continue;

-    message.is_system = true;
-    messageBlock.attr('is_system', String(true));
+        message.is_system = hide;
+        messageBlock.attr('is_system', String(hide));
+    }

    // Reload swipes. Useful when a last message is hidden.
    hideSwipeButtons();
@@ -69,28 +76,25 @@ export async function hideChatMessage(messageId, messageBlock) {
 }

 /**
- * Mark message as visible (non-system message).
+ * Mark message as hidden (system message).
+ * @deprecated Use hideChatMessageRange.
 * @param {number} messageId Message ID
- * @param {JQuery<Element>} messageBlock Message UI element
- * @returns
+ * @param {JQuery<Element>} _messageBlock Unused
+ * @returns {Promise<void>}
 */
-export async function unhideChatMessage(messageId, messageBlock) {
-    const chatId = getCurrentChatId();
+export async function hideChatMessage(messageId, _messageBlock) {
+    return hideChatMessageRange(messageId, messageId, false);
+}

-    if (!chatId || isNaN(messageId)) return;
-
-    const message = chat[messageId];
-
-    if (!message) return;
-
-    message.is_system = false;
-    messageBlock.attr('is_system', String(false));
-
-    // Reload swipes. Useful when a last message is hidden.
-    hideSwipeButtons();
-    showSwipeButtons();
-
-    saveChatDebounced();
+/**
+ * Mark message as visible (non-system message).
+ * @deprecated Use hideChatMessageRange.
+ * @param {number} messageId Message ID
+ * @param {JQuery<Element>} _messageBlock Unused
+ * @returns {Promise<void>}
+ */
+export async function unhideChatMessage(messageId, _messageBlock) {
+    return hideChatMessageRange(messageId, messageId, true);
 }

 /**
@@ -476,13 +480,13 @@ jQuery(function () {
    $(document).on('click', '.mes_hide', async function () {
        const messageBlock = $(this).closest('.mes');
        const messageId = Number(messageBlock.attr('mesid'));
-        await hideChatMessage(messageId, messageBlock);
+        await hideChatMessageRange(messageId, messageId, false);
    });

    $(document).on('click', '.mes_unhide', async function () {
        const messageBlock = $(this).closest('.mes');
        const messageId = Number(messageBlock.attr('mesid'));
-        await unhideChatMessage(messageId, messageBlock);
+        await hideChatMessageRange(messageId, messageId, true);
    });

    $(document).on('click', '.mes_file_delete', async function () {
--- a/public/scripts/extensions/memory/index.js
+++ b/public/scripts/extensions/memory/index.js
@@ -19,7 +19,7 @@ import { is_group_generating, selected_group } from '../../group-chats.js';
 import { registerSlashCommand } from '../../slash-commands.js';
 import { loadMovingUIState } from '../../power-user.js';
 import { dragElement } from '../../RossAscends-mods.js';
-import { getTextTokens, getTokenCount, tokenizers } from '../../tokenizers.js';
+import { getTextTokens, getTokenCountAsync, tokenizers } from '../../tokenizers.js';
 export { MODULE_NAME };

 const MODULE_NAME = '1_memory';
@@ -129,7 +129,7 @@ async function onPromptForceWordsAutoClick() {
    const allMessages = chat.filter(m => !m.is_system && m.mes).map(m => m.mes);
    const messagesWordCount = allMessages.map(m => extractAllWords(m)).flat().length;
    const averageMessageWordCount = messagesWordCount / allMessages.length;
-    const tokensPerWord = getTokenCount(allMessages.join('\n')) / messagesWordCount;
+    const tokensPerWord = await getTokenCountAsync(allMessages.join('\n')) / messagesWordCount;
    const wordsPerToken = 1 / tokensPerWord;
    const maxPromptLengthWords = Math.round(maxPromptLength * wordsPerToken);
    // How many words should pass so that messages will start be dropped out of context;
@@ -166,11 +166,11 @@ async function onPromptIntervalAutoClick() {
    const chat = context.chat;
    const allMessages = chat.filter(m => !m.is_system && m.mes).map(m => m.mes);
    const messagesWordCount = allMessages.map(m => extractAllWords(m)).flat().length;
-    const messagesTokenCount = getTokenCount(allMessages.join('\n'));
+    const messagesTokenCount = await getTokenCountAsync(allMessages.join('\n'));
    const tokensPerWord = messagesTokenCount / messagesWordCount;
    const averageMessageTokenCount = messagesTokenCount / allMessages.length;
    const targetSummaryTokens = Math.round(extension_settings.memory.promptWords * tokensPerWord);
-    const promptTokens = getTokenCount(extension_settings.memory.prompt);
+    const promptTokens = await getTokenCountAsync(extension_settings.memory.prompt);
    const promptAllowance = maxPromptLength - promptTokens - targetSummaryTokens;
    const maxMessagesPerSummary = extension_settings.memory.maxMessagesPerRequest || 0;
    const averageMessagesPerPrompt = Math.floor(promptAllowance / averageMessageTokenCount);
@@ -603,8 +603,7 @@ async function getRawSummaryPrompt(context, prompt) {
        const entry = `${message.name}:\n${message.mes}`;
        chatBuffer.push(entry);

-        const tokens = getTokenCount(getMemoryString(true), PADDING);
-        await delay(1);
+        const tokens = await getTokenCountAsync(getMemoryString(true), PADDING);

        if (tokens > PROMPT_SIZE) {
            chatBuffer.pop();
--- a/public/scripts/extensions/token-counter/index.js
+++ b/public/scripts/extensions/token-counter/index.js
@@ -1,7 +1,7 @@
 import { callPopup, main_api } from '../../../script.js';
 import { getContext } from '../../extensions.js';
 import { registerSlashCommand } from '../../slash-commands.js';
-import { getFriendlyTokenizerName, getTextTokens, getTokenCount, tokenizers } from '../../tokenizers.js';
+import { getFriendlyTokenizerName, getTextTokens, getTokenCountAsync, tokenizers } from '../../tokenizers.js';
 import { resetScrollHeight, debounce } from '../../utils.js';

 function rgb2hex(rgb) {
@@ -38,7 +38,7 @@ async function doTokenCounter() {
    </div>`;

    const dialog = $(html);
-    const countDebounced = debounce(() => {
+    const countDebounced = debounce(async () => {
        const text = String($('#token_counter_textarea').val());
        const ids = main_api == 'openai' ? getTextTokens(tokenizers.OPENAI, text) : getTextTokens(tokenizerId, text);

@@ -50,8 +50,7 @@ async function doTokenCounter() {
                drawChunks(Object.getOwnPropertyDescriptor(ids, 'chunks').value, ids);
            }
        } else {
-            const context = getContext();
-            const count = context.getTokenCount(text);
+            const count = await getTokenCountAsync(text);
            $('#token_counter_ids').text('—');
            $('#token_counter_result').text(count);
            $('#tokenized_chunks_display').text('—');
@@ -109,7 +108,7 @@ function drawChunks(chunks, ids) {
    }
 }

-function doCount() {
+async function doCount() {
    // get all of the messages in the chat
    const context = getContext();
    const messages = context.chat.filter(x => x.mes && !x.is_system).map(x => x.mes);
@@ -120,7 +119,8 @@ function doCount() {
    console.debug('All messages:', allMessages);

    //toastr success with the token count of the chat
-    toastr.success(`Token count: ${getTokenCount(allMessages)}`);
+    const count = await getTokenCountAsync(allMessages);
+    toastr.success(`Token count: ${count}`);
 }

 jQuery(() => {
--- a/public/scripts/logprobs.js
+++ b/public/scripts/logprobs.js
@@ -221,7 +221,7 @@ function onAlternativeClicked(tokenLogprobs, alternative) {
    }

    if (getGeneratingApi() === 'openai') {
-        return callPopup(`<h3>Feature unavailable</h3><p>Due to API limitations, rerolling a token is not supported with OpenAI. Try switching to a different API.</p>`, 'text');
+        return callPopup('<h3>Feature unavailable</h3><p>Due to API limitations, rerolling a token is not supported with OpenAI. Try switching to a different API.</p>', 'text');
    }

    const { messageLogprobs, continueFrom } = getActiveMessageLogprobData();
@@ -261,7 +261,7 @@ function onPrefixClicked() {

 function checkGenerateReady() {
    if (is_send_press) {
-        toastr.warning(`Please wait for the current generation to complete.`);
+        toastr.warning('Please wait for the current generation to complete.');
        return false;
    }
    return true;
@@ -292,13 +292,13 @@ function onToggleLogprobsPanel() {
    } else {
        logprobsViewer.addClass('resizing');
        logprobsViewer.transition({
-                opacity: 0.0,
-                duration: animation_duration,
-            },
-            async function () {
-                await delay(50);
-                logprobsViewer.removeClass('resizing');
-            });
+            opacity: 0.0,
+            duration: animation_duration,
+        },
+        async function () {
+            await delay(50);
+            logprobsViewer.removeClass('resizing');
+        });
        setTimeout(function () {
            logprobsViewer.hide();
        }, animation_duration);
@@ -407,7 +407,7 @@ export function saveLogprobsForActiveMessage(logprobs, continueFrom) {
        messageLogprobs: logprobs,
        continueFrom,
        hash: getMessageHash(chat[msgId]),
-    }
+    };

    state.messageLogprobs.set(data.hash, data);

@@ -458,7 +458,7 @@ function convertTokenIdLogprobsToText(input) {

    // Flatten unique token IDs across all logprobs
    const tokenIds = Array.from(new Set(input.flatMap(logprobs =>
-        logprobs.topLogprobs.map(([token]) => token).concat(logprobs.token)
+        logprobs.topLogprobs.map(([token]) => token).concat(logprobs.token),
    )));

    // Submit token IDs to tokenizer to get token text, then build ID->text map
@@ -469,7 +469,7 @@ function convertTokenIdLogprobsToText(input) {
    input.forEach(logprobs => {
        logprobs.token = tokenIdText.get(logprobs.token);
        logprobs.topLogprobs = logprobs.topLogprobs.map(([token, logprob]) =>
-            [tokenIdText.get(token), logprob]
+            [tokenIdText.get(token), logprob],
        );
    });
 }
--- a/public/scripts/openai.js
+++ b/public/scripts/openai.js
@@ -42,7 +42,7 @@ import {
    promptManagerDefaultPromptOrders,
 } from './PromptManager.js';

-import { getCustomStoppingStrings, persona_description_positions, power_user } from './power-user.js';
+import { forceCharacterEditorTokenize, getCustomStoppingStrings, persona_description_positions, power_user } from './power-user.js';
 import { SECRET_KEYS, secret_state, writeSecret } from './secrets.js';

 import { getEventSourceStream } from './sse-stream.js';
@@ -2264,7 +2264,7 @@ export class ChatCompletion {

            const shouldSquash = (message) => {
                return !excludeList.includes(message.identifier) && message.role === 'system' && !message.name;
-            }
+            };

            if (shouldSquash(message)) {
                if (lastMessage && shouldSquash(lastMessage)) {
@@ -3566,7 +3566,7 @@ async function onModelChange() {

    if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {
        if (oai_settings.max_context_unlocked) {
-            $('#openai_max_context').attr('max', unlocked_max);
+            $('#openai_max_context').attr('max', max_1mil);
        } else if (value === 'gemini-1.5-pro-latest') {
            $('#openai_max_context').attr('max', max_1mil);
        } else if (value === 'gemini-ultra' || value === 'gemini-1.0-pro-latest' || value === 'gemini-pro' || value === 'gemini-1.0-ultra-latest') {
@@ -4429,6 +4429,7 @@ $(document).ready(async function () {
        toggleChatCompletionForms();
        saveSettingsDebounced();
        reconnectOpenAi();
+        forceCharacterEditorTokenize();
        eventSource.emit(event_types.CHATCOMPLETION_SOURCE_CHANGED, oai_settings.chat_completion_source);
    });

--- a/public/scripts/personas.js
+++ b/public/scripts/personas.js
@@ -17,7 +17,7 @@ import {
    user_avatar,
 } from '../script.js';
 import { persona_description_positions, power_user } from './power-user.js';
-import { getTokenCount } from './tokenizers.js';
+import { getTokenCountAsync } from './tokenizers.js';
 import { debounce, delay, download, parseJsonFile } from './utils.js';

 const GRID_STORAGE_KEY = 'Personas_GridView';
@@ -171,9 +171,9 @@ export async function convertCharacterToPersona(characterId = null) {
 /**
 * Counts the number of tokens in a persona description.
 */
-const countPersonaDescriptionTokens = debounce(() => {
+const countPersonaDescriptionTokens = debounce(async () => {
    const description = String($('#persona_description').val());
-    const count = getTokenCount(description);
+    const count = await getTokenCountAsync(description);
    $('#persona_description_token_count').text(String(count));
 }, 1000);

--- a/public/scripts/popup.js
+++ b/public/scripts/popup.js
@@ -71,7 +71,7 @@ export class Popup {
        this.ok.textContent = okButton ?? 'OK';
        this.cancel.textContent = cancelButton ?? 'Cancel';

-        switch(type) {
+        switch (type) {
            case POPUP_TYPE.TEXT: {
                this.input.style.display = 'none';
                this.cancel.style.display = 'none';
@@ -107,9 +107,16 @@ export class Popup {
            // illegal argument
        }

-        this.ok.addEventListener('click', ()=>this.completeAffirmative());
-        this.cancel.addEventListener('click', ()=>this.completeNegative());
-        const keyListener = (evt)=>{
+        this.input.addEventListener('keydown', (evt) => {
+            if (evt.key != 'Enter' || evt.altKey || evt.ctrlKey || evt.shiftKey) return;
+            evt.preventDefault();
+            evt.stopPropagation();
+            this.completeAffirmative();
+        });
+
+        this.ok.addEventListener('click', () => this.completeAffirmative());
+        this.cancel.addEventListener('click', () => this.completeNegative());
+        const keyListener = (evt) => {
            switch (evt.key) {
                case 'Escape': {
                    evt.preventDefault();
@@ -127,7 +134,7 @@ export class Popup {
    async show() {
        document.body.append(this.dom);
        this.dom.style.display = 'block';
-        switch(this.type) {
+        switch (this.type) {
            case POPUP_TYPE.INPUT: {
                this.input.focus();
                break;
@@ -196,7 +203,7 @@ export class Popup {
            duration: animation_duration,
            easing: animation_easing,
        });
-        delay(animation_duration).then(()=>{
+        delay(animation_duration).then(() => {
            this.dom.remove();
        });

@@ -219,7 +226,7 @@ export function callGenericPopup(text, type, inputValue = '', { okButton, cancel
        text,
        type,
        inputValue,
-        { okButton, rows, wide, large, allowHorizontalScrolling, allowVerticalScrolling },
+        { okButton, cancelButton, rows, wide, large, allowHorizontalScrolling, allowVerticalScrolling },
    );
    return popup.show();
 }
--- a/public/scripts/power-user.js
+++ b/public/scripts/power-user.js
@@ -2764,6 +2764,14 @@ export function getCustomStoppingStrings(limit = undefined) {
    return strings;
 }

+export function forceCharacterEditorTokenize() {
+    $('[data-token-counter]').each(function () {
+        $(document.getElementById($(this).data('token-counter'))).data('last-value-hash', '');
+    });
+    $('#rm_ch_create_block').trigger('input');
+    $('#character_popup').trigger('input');
+}
+
 $(document).ready(() => {
    const adjustAutocompleteDebounced = debounce(() => {
        $('.ui-autocomplete-input').each(function () {
@@ -3175,8 +3183,7 @@ $(document).ready(() => {
        saveSettingsDebounced();

        // Trigger character editor re-tokenize
-        $('#rm_ch_create_block').trigger('input');
-        $('#character_popup').trigger('input');
+        forceCharacterEditorTokenize();
    });

    $('#send_on_enter').on('change', function () {
--- a/public/scripts/slash-commands.js
+++ b/public/scripts/slash-commands.js
@@ -38,7 +38,7 @@ import {
    this_chid,
 } from '../script.js';
 import { getMessageTimeStamp } from './RossAscends-mods.js';
-import { hideChatMessage, unhideChatMessage } from './chats.js';
+import { hideChatMessageRange } from './chats.js';
 import { getContext, saveMetadataDebounced } from './extensions.js';
 import { getRegexedString, regex_placement } from './extensions/regex/engine.js';
 import { findGroupMemberId, groups, is_group_generating, openGroupById, resetSelectedGroup, saveGroupChat, selected_group } from './group-chats.js';
@@ -46,7 +46,7 @@ import { chat_completion_sources, oai_settings } from './openai.js';
 import { autoSelectPersona } from './personas.js';
 import { addEphemeralStoppingString, chat_styles, flushEphemeralStoppingStrings, power_user } from './power-user.js';
 import { textgen_types, textgenerationwebui_settings } from './textgen-settings.js';
-import { decodeTextTokens, getFriendlyTokenizerName, getTextTokens, getTokenCount } from './tokenizers.js';
+import { decodeTextTokens, getFriendlyTokenizerName, getTextTokens, getTokenCountAsync } from './tokenizers.js';
 import { delay, isFalseBoolean, isTrueBoolean, stringToRange, trimToEndSentence, trimToStartSentence, waitUntilCondition } from './utils.js';
 import { registerVariableCommands, resolveVariable } from './variables.js';
 import { background_settings } from './backgrounds.js';
@@ -249,7 +249,7 @@ parser.addCommand('trimend', trimEndCallback, [], '<span class="monospace">(text
 parser.addCommand('inject', injectCallback, [], '<span class="monospace">id=injectId (position=before/after/chat depth=number scan=true/false role=system/user/assistant [text])</span> – injects a text into the LLM prompt for the current chat. Requires a unique injection ID. Positions: "before" main prompt, "after" main prompt, in-"chat" (default: after). Depth: injection depth for the prompt (default: 4). Role: role for in-chat injections (default: system). Scan: include injection content into World Info scans (default: false).', true, true);
 parser.addCommand('listinjects', listInjectsCallback, [], ' – lists all script injections for the current chat.', true, true);
 parser.addCommand('flushinjects', flushInjectsCallback, [], ' – removes all script injections for the current chat.', true, true);
-parser.addCommand('tokens', (_, text) => getTokenCount(text), [], '<span class="monospace">(text)</span> – counts the number of tokens in the text.', true, true);
+parser.addCommand('tokens', (_, text) => getTokenCountAsync(text), [], '<span class="monospace">(text)</span> – counts the number of tokens in the text.', true, true);
 parser.addCommand('model', modelCallback, [], '<span class="monospace">(model name)</span> – sets the model for the current API. Gets the current model name if no argument is provided.', true, true);
 registerVariableCommands();

@@ -388,7 +388,7 @@ function trimEndCallback(_, value) {
    return trimToEndSentence(value);
 }

-function trimTokensCallback(arg, value) {
+async function trimTokensCallback(arg, value) {
    if (!value) {
        console.warn('WARN: No argument provided for /trimtokens command');
        return '';
@@ -406,7 +406,7 @@ function trimTokensCallback(arg, value) {
    }

    const direction = arg.direction || 'end';
-    const tokenCount = getTokenCount(value);
+    const tokenCount = await getTokenCountAsync(value);

    // Token count is less than the limit, do nothing
    if (tokenCount <= limit) {
@@ -917,16 +917,7 @@ async function hideMessageCallback(_, arg) {
        return;
    }

-    for (let messageId = range.start; messageId <= range.end; messageId++) {
-        const messageBlock = $(`.mes[mesid="${messageId}"]`);
-
-        if (!messageBlock.length) {
-            console.warn(`WARN: No message found with ID ${messageId}`);
-            return;
-        }
-
-        await hideChatMessage(messageId, messageBlock);
-    }
+    await hideChatMessageRange(range.start, range.end, false);
 }

 async function unhideMessageCallback(_, arg) {
@@ -942,17 +933,7 @@ async function unhideMessageCallback(_, arg) {
        return '';
    }

-    for (let messageId = range.start; messageId <= range.end; messageId++) {
-        const messageBlock = $(`.mes[mesid="${messageId}"]`);
-
-        if (!messageBlock.length) {
-            console.warn(`WARN: No message found with ID ${messageId}`);
-            return '';
-        }
-
-        await unhideChatMessage(messageId, messageBlock);
-    }
-
+    await hideChatMessageRange(range.start, range.end, true);
    return '';
 }

--- a/public/scripts/tokenizers.js
+++ b/public/scripts/tokenizers.js
@@ -256,11 +256,93 @@ function callTokenizer(type, str) {
    }
 }

+/**
+ * Calls the underlying tokenizer model to the token count for a string.
+ * @param {number} type Tokenizer type.
+ * @param {string} str String to tokenize.
+ * @returns {Promise<number>} Token count.
+ */
+function callTokenizerAsync(type, str) {
+    return new Promise(resolve => {
+        if (type === tokenizers.NONE) {
+            return resolve(guesstimate(str));
+        }
+
+        switch (type) {
+            case tokenizers.API_CURRENT:
+                return callTokenizerAsync(currentRemoteTokenizerAPI(), str).then(resolve);
+            case tokenizers.API_KOBOLD:
+                return countTokensFromKoboldAPI(str, resolve);
+            case tokenizers.API_TEXTGENERATIONWEBUI:
+                return countTokensFromTextgenAPI(str, resolve);
+            default: {
+                const endpointUrl = TOKENIZER_URLS[type]?.count;
+                if (!endpointUrl) {
+                    console.warn('Unknown tokenizer type', type);
+                    return resolve(apiFailureTokenCount(str));
+                }
+                return countTokensFromServer(endpointUrl, str, resolve);
+            }
+        }
+    });
+}
+
+/**
+ * Gets the token count for a string using the current model tokenizer.
+ * @param {string} str String to tokenize
+ * @param {number | undefined} padding Optional padding tokens. Defaults to 0.
+ * @returns {Promise<number>} Token count.
+ */
+export async function getTokenCountAsync(str, padding = undefined) {
+    if (typeof str !== 'string' || !str?.length) {
+        return 0;
+    }
+
+    let tokenizerType = power_user.tokenizer;
+
+    if (main_api === 'openai') {
+        if (padding === power_user.token_padding) {
+            // For main "shadow" prompt building
+            tokenizerType = tokenizers.NONE;
+        } else {
+            // For extensions and WI
+            return counterWrapperOpenAIAsync(str);
+        }
+    }
+
+    if (tokenizerType === tokenizers.BEST_MATCH) {
+        tokenizerType = getTokenizerBestMatch(main_api);
+    }
+
+    if (padding === undefined) {
+        padding = 0;
+    }
+
+    const cacheObject = getTokenCacheObject();
+    const hash = getStringHash(str);
+    const cacheKey = `${tokenizerType}-${hash}+${padding}`;
+
+    if (typeof cacheObject[cacheKey] === 'number') {
+        return cacheObject[cacheKey];
+    }
+
+    const result = (await callTokenizerAsync(tokenizerType, str)) + padding;
+
+    if (isNaN(result)) {
+        console.warn('Token count calculation returned NaN');
+        return 0;
+    }
+
+    cacheObject[cacheKey] = result;
+    return result;
+}
+
 /**
 * Gets the token count for a string using the current model tokenizer.
 * @param {string} str String to tokenize
 * @param {number | undefined} padding Optional padding tokens. Defaults to 0.
 * @returns {number} Token count.
+ * @deprecated Use getTokenCountAsync instead.
 */
 export function getTokenCount(str, padding = undefined) {
    if (typeof str !== 'string' || !str?.length) {
@@ -310,12 +392,23 @@ export function getTokenCount(str, padding = undefined) {
 * Gets the token count for a string using the OpenAI tokenizer.
 * @param {string} text Text to tokenize.
 * @returns {number} Token count.
+ * @deprecated Use counterWrapperOpenAIAsync instead.
 */
 function counterWrapperOpenAI(text) {
    const message = { role: 'system', content: text };
    return countTokensOpenAI(message, true);
 }

+/**
+ * Gets the token count for a string using the OpenAI tokenizer.
+ * @param {string} text Text to tokenize.
+ * @returns {Promise<number>} Token count.
+ */
+function counterWrapperOpenAIAsync(text) {
+    const message = { role: 'system', content: text };
+    return countTokensOpenAIAsync(message, true);
+}
+
 export function getTokenizerModel() {
    // OpenAI models always provide their own tokenizer
    if (oai_settings.chat_completion_source == chat_completion_sources.OPENAI) {
@@ -410,6 +503,7 @@ export function getTokenizerModel() {

 /**
 * @param {any[] | Object} messages
+ * @deprecated Use countTokensOpenAIAsync instead.
 */
 export function countTokensOpenAI(messages, full = false) {
    const shouldTokenizeAI21 = oai_settings.chat_completion_source === chat_completion_sources.AI21 && oai_settings.use_ai21_tokenizer;
@@ -466,6 +560,66 @@ export function countTokensOpenAI(messages, full = false) {
    return token_count;
 }

+/**
+ * Returns the token count for a message using the OpenAI tokenizer.
+ * @param {object[]|object} messages
+ * @param {boolean} full
+ * @returns {Promise<number>} Token count.
+ */
+export async function countTokensOpenAIAsync(messages, full = false) {
+    const shouldTokenizeAI21 = oai_settings.chat_completion_source === chat_completion_sources.AI21 && oai_settings.use_ai21_tokenizer;
+    const shouldTokenizeGoogle = oai_settings.chat_completion_source === chat_completion_sources.MAKERSUITE && oai_settings.use_google_tokenizer;
+    let tokenizerEndpoint = '';
+    if (shouldTokenizeAI21) {
+        tokenizerEndpoint = '/api/tokenizers/ai21/count';
+    } else if (shouldTokenizeGoogle) {
+        tokenizerEndpoint = `/api/tokenizers/google/count?model=${getTokenizerModel()}`;
+    } else {
+        tokenizerEndpoint = `/api/tokenizers/openai/count?model=${getTokenizerModel()}`;
+    }
+    const cacheObject = getTokenCacheObject();
+
+    if (!Array.isArray(messages)) {
+        messages = [messages];
+    }
+
+    let token_count = -1;
+
+    for (const message of messages) {
+        const model = getTokenizerModel();
+
+        if (model === 'claude' || shouldTokenizeAI21 || shouldTokenizeGoogle) {
+            full = true;
+        }
+
+        const hash = getStringHash(JSON.stringify(message));
+        const cacheKey = `${model}-${hash}`;
+        const cachedCount = cacheObject[cacheKey];
+
+        if (typeof cachedCount === 'number') {
+            token_count += cachedCount;
+        }
+
+        else {
+            const data = await jQuery.ajax({
+                async: true,
+                type: 'POST', //
+                url: tokenizerEndpoint,
+                data: JSON.stringify([message]),
+                dataType: 'json',
+                contentType: 'application/json',
+            });
+
+            token_count += Number(data.token_count);
+            cacheObject[cacheKey] = Number(data.token_count);
+        }
+    }
+
+    if (!full) token_count -= 2;
+
+    return token_count;
+}
+
 /**
 * Gets the token cache object for the current chat.
 * @returns {Object} Token cache object for the current chat.
@@ -495,13 +649,15 @@ function getTokenCacheObject() {
 * Count tokens using the server API.
 * @param {string} endpoint API endpoint.
 * @param {string} str String to tokenize.
+ * @param {function} [resolve] Promise resolve function.s
 * @returns {number} Token count.
 */
-function countTokensFromServer(endpoint, str) {
+function countTokensFromServer(endpoint, str, resolve) {
+    const isAsync = typeof resolve === 'function';
    let tokenCount = 0;

    jQuery.ajax({
-        async: false,
+        async: isAsync,
        type: 'POST',
        url: endpoint,
        data: JSON.stringify({ text: str }),
@@ -513,6 +669,8 @@ function countTokensFromServer(endpoint, str) {
            } else {
                tokenCount = apiFailureTokenCount(str);
            }
+
+            isAsync && resolve(tokenCount);
        },
    });

@@ -522,13 +680,15 @@ function countTokensFromServer(endpoint, str) {
 /**
 * Count tokens using the AI provider's API.
 * @param {string} str String to tokenize.
+ * @param {function} [resolve] Promise resolve function.
 * @returns {number} Token count.
 */
-function countTokensFromKoboldAPI(str) {
+function countTokensFromKoboldAPI(str, resolve) {
+    const isAsync = typeof resolve === 'function';
    let tokenCount = 0;

    jQuery.ajax({
-        async: false,
+        async: isAsync,
        type: 'POST',
        url: TOKENIZER_URLS[tokenizers.API_KOBOLD].count,
        data: JSON.stringify({
@@ -543,6 +703,8 @@ function countTokensFromKoboldAPI(str) {
            } else {
                tokenCount = apiFailureTokenCount(str);
            }
+
+            isAsync && resolve(tokenCount);
        },
    });

@@ -561,13 +723,15 @@ function getTextgenAPITokenizationParams(str) {
 /**
 * Count tokens using the AI provider's API.
 * @param {string} str String to tokenize.
+ * @param {function} [resolve] Promise resolve function.
 * @returns {number} Token count.
 */
-function countTokensFromTextgenAPI(str) {
+function countTokensFromTextgenAPI(str, resolve) {
+    const isAsync = typeof resolve === 'function';
    let tokenCount = 0;

    jQuery.ajax({
-        async: false,
+        async: isAsync,
        type: 'POST',
        url: TOKENIZER_URLS[tokenizers.API_TEXTGENERATIONWEBUI].count,
        data: JSON.stringify(getTextgenAPITokenizationParams(str)),
@@ -579,6 +743,8 @@ function countTokensFromTextgenAPI(str) {
            } else {
                tokenCount = apiFailureTokenCount(str);
            }
+
+            isAsync && resolve(tokenCount);
        },
    });

@@ -605,12 +771,14 @@ function apiFailureTokenCount(str) {
 * Calls the underlying tokenizer model to encode a string to tokens.
 * @param {string} endpoint API endpoint.
 * @param {string} str String to tokenize.
+ * @param {function} [resolve] Promise resolve function.
 * @returns {number[]} Array of token ids.
 */
-function getTextTokensFromServer(endpoint, str) {
+function getTextTokensFromServer(endpoint, str, resolve) {
+    const isAsync = typeof resolve === 'function';
    let ids = [];
    jQuery.ajax({
-        async: false,
+        async: isAsync,
        type: 'POST',
        url: endpoint,
        data: JSON.stringify({ text: str }),
@@ -623,6 +791,8 @@ function getTextTokensFromServer(endpoint, str) {
            if (Array.isArray(data.chunks)) {
                Object.defineProperty(ids, 'chunks', { value: data.chunks });
            }
+
+            isAsync && resolve(ids);
        },
    });
    return ids;
@@ -631,12 +801,14 @@ function getTextTokensFromServer(endpoint, str) {
 /**
 * Calls the AI provider's tokenize API to encode a string to tokens.
 * @param {string} str String to tokenize.
+ * @param {function} [resolve] Promise resolve function.
 * @returns {number[]} Array of token ids.
 */
-function getTextTokensFromTextgenAPI(str) {
+function getTextTokensFromTextgenAPI(str, resolve) {
+    const isAsync = typeof resolve === 'function';
    let ids = [];
    jQuery.ajax({
-        async: false,
+        async: isAsync,
        type: 'POST',
        url: TOKENIZER_URLS[tokenizers.API_TEXTGENERATIONWEBUI].encode,
        data: JSON.stringify(getTextgenAPITokenizationParams(str)),
@@ -644,6 +816,7 @@ function getTextTokensFromTextgenAPI(str) {
        contentType: 'application/json',
        success: function (data) {
            ids = data.ids;
+            isAsync && resolve(ids);
        },
    });
    return ids;
@@ -652,13 +825,15 @@ function getTextTokensFromTextgenAPI(str) {
 /**
 * Calls the AI provider's tokenize API to encode a string to tokens.
 * @param {string} str String to tokenize.
+ * @param {function} [resolve] Promise resolve function.
 * @returns {number[]} Array of token ids.
 */
-function getTextTokensFromKoboldAPI(str) {
+function getTextTokensFromKoboldAPI(str, resolve) {
+    const isAsync = typeof resolve === 'function';
    let ids = [];

    jQuery.ajax({
-        async: false,
+        async: isAsync,
        type: 'POST',
        url: TOKENIZER_URLS[tokenizers.API_KOBOLD].encode,
        data: JSON.stringify({
@@ -669,6 +844,7 @@ function getTextTokensFromKoboldAPI(str) {
        contentType: 'application/json',
        success: function (data) {
            ids = data.ids;
+            isAsync && resolve(ids);
        },
    });

@@ -679,13 +855,15 @@ function getTextTokensFromKoboldAPI(str) {
 * Calls the underlying tokenizer model to decode token ids to text.
 * @param {string} endpoint API endpoint.
 * @param {number[]} ids Array of token ids
+ * @param {function} [resolve] Promise resolve function.
 * @returns {({ text: string, chunks?: string[] })} Decoded token text as a single string and individual chunks (if available).
 */
-function decodeTextTokensFromServer(endpoint, ids) {
+function decodeTextTokensFromServer(endpoint, ids, resolve) {
+    const isAsync = typeof resolve === 'function';
    let text = '';
    let chunks = [];
    jQuery.ajax({
-        async: false,
+        async: isAsync,
        type: 'POST',
        url: endpoint,
        data: JSON.stringify({ ids: ids }),
@@ -694,6 +872,7 @@ function decodeTextTokensFromServer(endpoint, ids) {
        success: function (data) {
            text = data.text;
            chunks = data.chunks;
+            isAsync && resolve({ text, chunks });
        },
    });
    return { text, chunks };
--- a/public/scripts/world-info.js
+++ b/public/scripts/world-info.js
@@ -5,7 +5,7 @@ import { NOTE_MODULE_NAME, metadata_keys, shouldWIAddPrompt } from './authors-no
 import { registerSlashCommand } from './slash-commands.js';
 import { isMobile } from './RossAscends-mods.js';
 import { FILTER_TYPES, FilterHelper } from './filters.js';
-import { getTokenCount } from './tokenizers.js';
+import { getTokenCountAsync } from './tokenizers.js';
 import { power_user } from './power-user.js';
 import { getTagKeyForEntity } from './tags.js';
 import { resolveVariable } from './variables.js';
@@ -1189,8 +1189,8 @@ function getWorldEntry(name, data, entry) {

    // content
    const counter = template.find('.world_entry_form_token_counter');
-    const countTokensDebounced = debounce(function (counter, value) {
-        const numberOfTokens = getTokenCount(value);
+    const countTokensDebounced = debounce(async function (counter, value) {
+        const numberOfTokens = await getTokenCountAsync(value);
        $(counter).text(numberOfTokens);
    }, 1000);

@@ -2177,7 +2177,7 @@ async function checkWorldInfo(chat, maxContext) {
        const newEntries = [...activatedNow]
            .sort((a, b) => sortedEntries.indexOf(a) - sortedEntries.indexOf(b));
        let newContent = '';
-        const textToScanTokens = getTokenCount(allActivatedText);
+        const textToScanTokens = await getTokenCountAsync(allActivatedText);
        const probabilityChecksBefore = failedProbabilityChecks.size;

        filterByInclusionGroups(newEntries, allActivatedEntries);
@@ -2194,7 +2194,7 @@ async function checkWorldInfo(chat, maxContext) {

            newContent += `${substituteParams(entry.content)}\n`;

-            if (textToScanTokens + getTokenCount(newContent) >= budget) {
+            if ((textToScanTokens + (await getTokenCountAsync(newContent))) >= budget) {
                console.debug('WI budget reached, stopping');
                if (world_info_overflow_alert) {
                    console.log('Alerting');