}
+ */
+export async function unhideChatMessage(messageId, _messageBlock) {
+ return hideChatMessageRange(messageId, messageId, true);
}
/**
@@ -476,13 +480,13 @@ jQuery(function () {
$(document).on('click', '.mes_hide', async function () {
const messageBlock = $(this).closest('.mes');
const messageId = Number(messageBlock.attr('mesid'));
- await hideChatMessage(messageId, messageBlock);
+ await hideChatMessageRange(messageId, messageId, false);
});
$(document).on('click', '.mes_unhide', async function () {
const messageBlock = $(this).closest('.mes');
const messageId = Number(messageBlock.attr('mesid'));
- await unhideChatMessage(messageId, messageBlock);
+ await hideChatMessageRange(messageId, messageId, true);
});
$(document).on('click', '.mes_file_delete', async function () {
diff --git a/public/scripts/extensions/memory/index.js b/public/scripts/extensions/memory/index.js
index ff0dea958..323f57e80 100644
--- a/public/scripts/extensions/memory/index.js
+++ b/public/scripts/extensions/memory/index.js
@@ -19,7 +19,7 @@ import { is_group_generating, selected_group } from '../../group-chats.js';
import { registerSlashCommand } from '../../slash-commands.js';
import { loadMovingUIState } from '../../power-user.js';
import { dragElement } from '../../RossAscends-mods.js';
-import { getTextTokens, getTokenCount, tokenizers } from '../../tokenizers.js';
+import { getTextTokens, getTokenCountAsync, tokenizers } from '../../tokenizers.js';
export { MODULE_NAME };
const MODULE_NAME = '1_memory';
@@ -129,7 +129,7 @@ async function onPromptForceWordsAutoClick() {
const allMessages = chat.filter(m => !m.is_system && m.mes).map(m => m.mes);
const messagesWordCount = allMessages.map(m => extractAllWords(m)).flat().length;
const averageMessageWordCount = messagesWordCount / allMessages.length;
- const tokensPerWord = getTokenCount(allMessages.join('\n')) / messagesWordCount;
+ const tokensPerWord = await getTokenCountAsync(allMessages.join('\n')) / messagesWordCount;
const wordsPerToken = 1 / tokensPerWord;
const maxPromptLengthWords = Math.round(maxPromptLength * wordsPerToken);
// How many words should pass so that messages will start be dropped out of context;
@@ -166,11 +166,11 @@ async function onPromptIntervalAutoClick() {
const chat = context.chat;
const allMessages = chat.filter(m => !m.is_system && m.mes).map(m => m.mes);
const messagesWordCount = allMessages.map(m => extractAllWords(m)).flat().length;
- const messagesTokenCount = getTokenCount(allMessages.join('\n'));
+ const messagesTokenCount = await getTokenCountAsync(allMessages.join('\n'));
const tokensPerWord = messagesTokenCount / messagesWordCount;
const averageMessageTokenCount = messagesTokenCount / allMessages.length;
const targetSummaryTokens = Math.round(extension_settings.memory.promptWords * tokensPerWord);
- const promptTokens = getTokenCount(extension_settings.memory.prompt);
+ const promptTokens = await getTokenCountAsync(extension_settings.memory.prompt);
const promptAllowance = maxPromptLength - promptTokens - targetSummaryTokens;
const maxMessagesPerSummary = extension_settings.memory.maxMessagesPerRequest || 0;
const averageMessagesPerPrompt = Math.floor(promptAllowance / averageMessageTokenCount);
@@ -603,8 +603,7 @@ async function getRawSummaryPrompt(context, prompt) {
const entry = `${message.name}:\n${message.mes}`;
chatBuffer.push(entry);
- const tokens = getTokenCount(getMemoryString(true), PADDING);
- await delay(1);
+ const tokens = await getTokenCountAsync(getMemoryString(true), PADDING);
if (tokens > PROMPT_SIZE) {
chatBuffer.pop();
diff --git a/public/scripts/extensions/token-counter/index.js b/public/scripts/extensions/token-counter/index.js
index 90cdf9ee8..d9231dac1 100644
--- a/public/scripts/extensions/token-counter/index.js
+++ b/public/scripts/extensions/token-counter/index.js
@@ -1,7 +1,7 @@
import { callPopup, main_api } from '../../../script.js';
import { getContext } from '../../extensions.js';
import { registerSlashCommand } from '../../slash-commands.js';
-import { getFriendlyTokenizerName, getTextTokens, getTokenCount, tokenizers } from '../../tokenizers.js';
+import { getFriendlyTokenizerName, getTextTokens, getTokenCountAsync, tokenizers } from '../../tokenizers.js';
import { resetScrollHeight, debounce } from '../../utils.js';
function rgb2hex(rgb) {
@@ -38,7 +38,7 @@ async function doTokenCounter() {
`;
const dialog = $(html);
- const countDebounced = debounce(() => {
+ const countDebounced = debounce(async () => {
const text = String($('#token_counter_textarea').val());
const ids = main_api == 'openai' ? getTextTokens(tokenizers.OPENAI, text) : getTextTokens(tokenizerId, text);
@@ -50,8 +50,7 @@ async function doTokenCounter() {
drawChunks(Object.getOwnPropertyDescriptor(ids, 'chunks').value, ids);
}
} else {
- const context = getContext();
- const count = context.getTokenCount(text);
+ const count = await getTokenCountAsync(text);
$('#token_counter_ids').text('—');
$('#token_counter_result').text(count);
$('#tokenized_chunks_display').text('—');
@@ -109,7 +108,7 @@ function drawChunks(chunks, ids) {
}
}
-function doCount() {
+async function doCount() {
// get all of the messages in the chat
const context = getContext();
const messages = context.chat.filter(x => x.mes && !x.is_system).map(x => x.mes);
@@ -120,7 +119,8 @@ function doCount() {
console.debug('All messages:', allMessages);
//toastr success with the token count of the chat
- toastr.success(`Token count: ${getTokenCount(allMessages)}`);
+ const count = await getTokenCountAsync(allMessages);
+ toastr.success(`Token count: ${count}`);
}
jQuery(() => {
diff --git a/public/scripts/logprobs.js b/public/scripts/logprobs.js
index b2e682286..e87adf487 100644
--- a/public/scripts/logprobs.js
+++ b/public/scripts/logprobs.js
@@ -221,7 +221,7 @@ function onAlternativeClicked(tokenLogprobs, alternative) {
}
if (getGeneratingApi() === 'openai') {
- return callPopup(`Feature unavailable
Due to API limitations, rerolling a token is not supported with OpenAI. Try switching to a different API.
`, 'text');
+ return callPopup('Feature unavailable
Due to API limitations, rerolling a token is not supported with OpenAI. Try switching to a different API.
', 'text');
}
const { messageLogprobs, continueFrom } = getActiveMessageLogprobData();
@@ -261,7 +261,7 @@ function onPrefixClicked() {
function checkGenerateReady() {
if (is_send_press) {
- toastr.warning(`Please wait for the current generation to complete.`);
+ toastr.warning('Please wait for the current generation to complete.');
return false;
}
return true;
@@ -292,13 +292,13 @@ function onToggleLogprobsPanel() {
} else {
logprobsViewer.addClass('resizing');
logprobsViewer.transition({
- opacity: 0.0,
- duration: animation_duration,
- },
- async function () {
- await delay(50);
- logprobsViewer.removeClass('resizing');
- });
+ opacity: 0.0,
+ duration: animation_duration,
+ },
+ async function () {
+ await delay(50);
+ logprobsViewer.removeClass('resizing');
+ });
setTimeout(function () {
logprobsViewer.hide();
}, animation_duration);
@@ -407,7 +407,7 @@ export function saveLogprobsForActiveMessage(logprobs, continueFrom) {
messageLogprobs: logprobs,
continueFrom,
hash: getMessageHash(chat[msgId]),
- }
+ };
state.messageLogprobs.set(data.hash, data);
@@ -458,7 +458,7 @@ function convertTokenIdLogprobsToText(input) {
// Flatten unique token IDs across all logprobs
const tokenIds = Array.from(new Set(input.flatMap(logprobs =>
- logprobs.topLogprobs.map(([token]) => token).concat(logprobs.token)
+ logprobs.topLogprobs.map(([token]) => token).concat(logprobs.token),
)));
// Submit token IDs to tokenizer to get token text, then build ID->text map
@@ -469,7 +469,7 @@ function convertTokenIdLogprobsToText(input) {
input.forEach(logprobs => {
logprobs.token = tokenIdText.get(logprobs.token);
logprobs.topLogprobs = logprobs.topLogprobs.map(([token, logprob]) =>
- [tokenIdText.get(token), logprob]
+ [tokenIdText.get(token), logprob],
);
});
}
diff --git a/public/scripts/openai.js b/public/scripts/openai.js
index 8a71b89f0..b8d7d192e 100644
--- a/public/scripts/openai.js
+++ b/public/scripts/openai.js
@@ -42,7 +42,7 @@ import {
promptManagerDefaultPromptOrders,
} from './PromptManager.js';
-import { getCustomStoppingStrings, persona_description_positions, power_user } from './power-user.js';
+import { forceCharacterEditorTokenize, getCustomStoppingStrings, persona_description_positions, power_user } from './power-user.js';
import { SECRET_KEYS, secret_state, writeSecret } from './secrets.js';
import { getEventSourceStream } from './sse-stream.js';
@@ -2264,7 +2264,7 @@ export class ChatCompletion {
const shouldSquash = (message) => {
return !excludeList.includes(message.identifier) && message.role === 'system' && !message.name;
- }
+ };
if (shouldSquash(message)) {
if (lastMessage && shouldSquash(lastMessage)) {
@@ -3566,7 +3566,7 @@ async function onModelChange() {
if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {
if (oai_settings.max_context_unlocked) {
- $('#openai_max_context').attr('max', unlocked_max);
+ $('#openai_max_context').attr('max', max_1mil);
} else if (value === 'gemini-1.5-pro-latest') {
$('#openai_max_context').attr('max', max_1mil);
} else if (value === 'gemini-ultra' || value === 'gemini-1.0-pro-latest' || value === 'gemini-pro' || value === 'gemini-1.0-ultra-latest') {
@@ -4429,6 +4429,7 @@ $(document).ready(async function () {
toggleChatCompletionForms();
saveSettingsDebounced();
reconnectOpenAi();
+ forceCharacterEditorTokenize();
eventSource.emit(event_types.CHATCOMPLETION_SOURCE_CHANGED, oai_settings.chat_completion_source);
});
diff --git a/public/scripts/personas.js b/public/scripts/personas.js
index 084276331..aa2aadb3b 100644
--- a/public/scripts/personas.js
+++ b/public/scripts/personas.js
@@ -17,7 +17,7 @@ import {
user_avatar,
} from '../script.js';
import { persona_description_positions, power_user } from './power-user.js';
-import { getTokenCount } from './tokenizers.js';
+import { getTokenCountAsync } from './tokenizers.js';
import { debounce, delay, download, parseJsonFile } from './utils.js';
const GRID_STORAGE_KEY = 'Personas_GridView';
@@ -171,9 +171,9 @@ export async function convertCharacterToPersona(characterId = null) {
/**
* Counts the number of tokens in a persona description.
*/
-const countPersonaDescriptionTokens = debounce(() => {
+const countPersonaDescriptionTokens = debounce(async () => {
const description = String($('#persona_description').val());
- const count = getTokenCount(description);
+ const count = await getTokenCountAsync(description);
$('#persona_description_token_count').text(String(count));
}, 1000);
diff --git a/public/scripts/popup.js b/public/scripts/popup.js
index b793f3a66..4e75431f4 100644
--- a/public/scripts/popup.js
+++ b/public/scripts/popup.js
@@ -71,7 +71,7 @@ export class Popup {
this.ok.textContent = okButton ?? 'OK';
this.cancel.textContent = cancelButton ?? 'Cancel';
- switch(type) {
+ switch (type) {
case POPUP_TYPE.TEXT: {
this.input.style.display = 'none';
this.cancel.style.display = 'none';
@@ -107,9 +107,16 @@ export class Popup {
// illegal argument
}
- this.ok.addEventListener('click', ()=>this.completeAffirmative());
- this.cancel.addEventListener('click', ()=>this.completeNegative());
- const keyListener = (evt)=>{
+ this.input.addEventListener('keydown', (evt) => {
+ if (evt.key != 'Enter' || evt.altKey || evt.ctrlKey || evt.shiftKey) return;
+ evt.preventDefault();
+ evt.stopPropagation();
+ this.completeAffirmative();
+ });
+
+ this.ok.addEventListener('click', () => this.completeAffirmative());
+ this.cancel.addEventListener('click', () => this.completeNegative());
+ const keyListener = (evt) => {
switch (evt.key) {
case 'Escape': {
evt.preventDefault();
@@ -127,7 +134,7 @@ export class Popup {
async show() {
document.body.append(this.dom);
this.dom.style.display = 'block';
- switch(this.type) {
+ switch (this.type) {
case POPUP_TYPE.INPUT: {
this.input.focus();
break;
@@ -196,7 +203,7 @@ export class Popup {
duration: animation_duration,
easing: animation_easing,
});
- delay(animation_duration).then(()=>{
+ delay(animation_duration).then(() => {
this.dom.remove();
});
@@ -219,7 +226,7 @@ export function callGenericPopup(text, type, inputValue = '', { okButton, cancel
text,
type,
inputValue,
- { okButton, rows, wide, large, allowHorizontalScrolling, allowVerticalScrolling },
+ { okButton, cancelButton, rows, wide, large, allowHorizontalScrolling, allowVerticalScrolling },
);
return popup.show();
}
diff --git a/public/scripts/power-user.js b/public/scripts/power-user.js
index 2bf3169b5..35f4984b7 100644
--- a/public/scripts/power-user.js
+++ b/public/scripts/power-user.js
@@ -2764,6 +2764,14 @@ export function getCustomStoppingStrings(limit = undefined) {
return strings;
}
+export function forceCharacterEditorTokenize() {
+ $('[data-token-counter]').each(function () {
+ $(document.getElementById($(this).data('token-counter'))).data('last-value-hash', '');
+ });
+ $('#rm_ch_create_block').trigger('input');
+ $('#character_popup').trigger('input');
+}
+
$(document).ready(() => {
const adjustAutocompleteDebounced = debounce(() => {
$('.ui-autocomplete-input').each(function () {
@@ -3175,8 +3183,7 @@ $(document).ready(() => {
saveSettingsDebounced();
// Trigger character editor re-tokenize
- $('#rm_ch_create_block').trigger('input');
- $('#character_popup').trigger('input');
+ forceCharacterEditorTokenize();
});
$('#send_on_enter').on('change', function () {
diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js
index 8130488e7..b403205e9 100644
--- a/public/scripts/slash-commands.js
+++ b/public/scripts/slash-commands.js
@@ -38,7 +38,7 @@ import {
this_chid,
} from '../script.js';
import { getMessageTimeStamp } from './RossAscends-mods.js';
-import { hideChatMessage, unhideChatMessage } from './chats.js';
+import { hideChatMessageRange } from './chats.js';
import { getContext, saveMetadataDebounced } from './extensions.js';
import { getRegexedString, regex_placement } from './extensions/regex/engine.js';
import { findGroupMemberId, groups, is_group_generating, openGroupById, resetSelectedGroup, saveGroupChat, selected_group } from './group-chats.js';
@@ -46,7 +46,7 @@ import { chat_completion_sources, oai_settings } from './openai.js';
import { autoSelectPersona } from './personas.js';
import { addEphemeralStoppingString, chat_styles, flushEphemeralStoppingStrings, power_user } from './power-user.js';
import { textgen_types, textgenerationwebui_settings } from './textgen-settings.js';
-import { decodeTextTokens, getFriendlyTokenizerName, getTextTokens, getTokenCount } from './tokenizers.js';
+import { decodeTextTokens, getFriendlyTokenizerName, getTextTokens, getTokenCountAsync } from './tokenizers.js';
import { delay, isFalseBoolean, isTrueBoolean, stringToRange, trimToEndSentence, trimToStartSentence, waitUntilCondition } from './utils.js';
import { registerVariableCommands, resolveVariable } from './variables.js';
import { background_settings } from './backgrounds.js';
@@ -249,7 +249,7 @@ parser.addCommand('trimend', trimEndCallback, [], '(text
parser.addCommand('inject', injectCallback, [], 'id=injectId (position=before/after/chat depth=number scan=true/false role=system/user/assistant [text]) – injects a text into the LLM prompt for the current chat. Requires a unique injection ID. Positions: "before" main prompt, "after" main prompt, in-"chat" (default: after). Depth: injection depth for the prompt (default: 4). Role: role for in-chat injections (default: system). Scan: include injection content into World Info scans (default: false).', true, true);
parser.addCommand('listinjects', listInjectsCallback, [], ' – lists all script injections for the current chat.', true, true);
parser.addCommand('flushinjects', flushInjectsCallback, [], ' – removes all script injections for the current chat.', true, true);
-parser.addCommand('tokens', (_, text) => getTokenCount(text), [], '(text) – counts the number of tokens in the text.', true, true);
+parser.addCommand('tokens', (_, text) => getTokenCountAsync(text), [], '(text) – counts the number of tokens in the text.', true, true);
parser.addCommand('model', modelCallback, [], '(model name) – sets the model for the current API. Gets the current model name if no argument is provided.', true, true);
registerVariableCommands();
@@ -388,7 +388,7 @@ function trimEndCallback(_, value) {
return trimToEndSentence(value);
}
-function trimTokensCallback(arg, value) {
+async function trimTokensCallback(arg, value) {
if (!value) {
console.warn('WARN: No argument provided for /trimtokens command');
return '';
@@ -406,7 +406,7 @@ function trimTokensCallback(arg, value) {
}
const direction = arg.direction || 'end';
- const tokenCount = getTokenCount(value);
+ const tokenCount = await getTokenCountAsync(value);
// Token count is less than the limit, do nothing
if (tokenCount <= limit) {
@@ -917,16 +917,7 @@ async function hideMessageCallback(_, arg) {
return;
}
- for (let messageId = range.start; messageId <= range.end; messageId++) {
- const messageBlock = $(`.mes[mesid="${messageId}"]`);
-
- if (!messageBlock.length) {
- console.warn(`WARN: No message found with ID ${messageId}`);
- return;
- }
-
- await hideChatMessage(messageId, messageBlock);
- }
+ await hideChatMessageRange(range.start, range.end, false);
}
async function unhideMessageCallback(_, arg) {
@@ -942,17 +933,7 @@ async function unhideMessageCallback(_, arg) {
return '';
}
- for (let messageId = range.start; messageId <= range.end; messageId++) {
- const messageBlock = $(`.mes[mesid="${messageId}"]`);
-
- if (!messageBlock.length) {
- console.warn(`WARN: No message found with ID ${messageId}`);
- return '';
- }
-
- await unhideChatMessage(messageId, messageBlock);
- }
-
+ await hideChatMessageRange(range.start, range.end, true);
return '';
}
diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js
index 03ae0b7f4..7e9fc7856 100644
--- a/public/scripts/tokenizers.js
+++ b/public/scripts/tokenizers.js
@@ -256,11 +256,93 @@ function callTokenizer(type, str) {
}
}
+/**
+ * Calls the underlying tokenizer model to the token count for a string.
+ * @param {number} type Tokenizer type.
+ * @param {string} str String to tokenize.
+ * @returns {Promise} Token count.
+ */
+function callTokenizerAsync(type, str) {
+ return new Promise(resolve => {
+ if (type === tokenizers.NONE) {
+ return resolve(guesstimate(str));
+ }
+
+ switch (type) {
+ case tokenizers.API_CURRENT:
+ return callTokenizerAsync(currentRemoteTokenizerAPI(), str).then(resolve);
+ case tokenizers.API_KOBOLD:
+ return countTokensFromKoboldAPI(str, resolve);
+ case tokenizers.API_TEXTGENERATIONWEBUI:
+ return countTokensFromTextgenAPI(str, resolve);
+ default: {
+ const endpointUrl = TOKENIZER_URLS[type]?.count;
+ if (!endpointUrl) {
+ console.warn('Unknown tokenizer type', type);
+ return resolve(apiFailureTokenCount(str));
+ }
+ return countTokensFromServer(endpointUrl, str, resolve);
+ }
+ }
+ });
+}
+
+/**
+ * Gets the token count for a string using the current model tokenizer.
+ * @param {string} str String to tokenize
+ * @param {number | undefined} padding Optional padding tokens. Defaults to 0.
+ * @returns {Promise} Token count.
+ */
+export async function getTokenCountAsync(str, padding = undefined) {
+ if (typeof str !== 'string' || !str?.length) {
+ return 0;
+ }
+
+ let tokenizerType = power_user.tokenizer;
+
+ if (main_api === 'openai') {
+ if (padding === power_user.token_padding) {
+ // For main "shadow" prompt building
+ tokenizerType = tokenizers.NONE;
+ } else {
+ // For extensions and WI
+ return counterWrapperOpenAIAsync(str);
+ }
+ }
+
+ if (tokenizerType === tokenizers.BEST_MATCH) {
+ tokenizerType = getTokenizerBestMatch(main_api);
+ }
+
+ if (padding === undefined) {
+ padding = 0;
+ }
+
+ const cacheObject = getTokenCacheObject();
+ const hash = getStringHash(str);
+ const cacheKey = `${tokenizerType}-${hash}+${padding}`;
+
+ if (typeof cacheObject[cacheKey] === 'number') {
+ return cacheObject[cacheKey];
+ }
+
+ const result = (await callTokenizerAsync(tokenizerType, str)) + padding;
+
+ if (isNaN(result)) {
+ console.warn('Token count calculation returned NaN');
+ return 0;
+ }
+
+ cacheObject[cacheKey] = result;
+ return result;
+}
+
/**
* Gets the token count for a string using the current model tokenizer.
* @param {string} str String to tokenize
* @param {number | undefined} padding Optional padding tokens. Defaults to 0.
* @returns {number} Token count.
+ * @deprecated Use getTokenCountAsync instead.
*/
export function getTokenCount(str, padding = undefined) {
if (typeof str !== 'string' || !str?.length) {
@@ -310,12 +392,23 @@ export function getTokenCount(str, padding = undefined) {
* Gets the token count for a string using the OpenAI tokenizer.
* @param {string} text Text to tokenize.
* @returns {number} Token count.
+ * @deprecated Use counterWrapperOpenAIAsync instead.
*/
function counterWrapperOpenAI(text) {
const message = { role: 'system', content: text };
return countTokensOpenAI(message, true);
}
+/**
+ * Gets the token count for a string using the OpenAI tokenizer.
+ * @param {string} text Text to tokenize.
+ * @returns {Promise} Token count.
+ */
+function counterWrapperOpenAIAsync(text) {
+ const message = { role: 'system', content: text };
+ return countTokensOpenAIAsync(message, true);
+}
+
export function getTokenizerModel() {
// OpenAI models always provide their own tokenizer
if (oai_settings.chat_completion_source == chat_completion_sources.OPENAI) {
@@ -410,6 +503,7 @@ export function getTokenizerModel() {
/**
* @param {any[] | Object} messages
+ * @deprecated Use countTokensOpenAIAsync instead.
*/
export function countTokensOpenAI(messages, full = false) {
const shouldTokenizeAI21 = oai_settings.chat_completion_source === chat_completion_sources.AI21 && oai_settings.use_ai21_tokenizer;
@@ -466,6 +560,66 @@ export function countTokensOpenAI(messages, full = false) {
return token_count;
}
+/**
+ * Returns the token count for a message using the OpenAI tokenizer.
+ * @param {object[]|object} messages
+ * @param {boolean} full
+ * @returns {Promise} Token count.
+ */
+export async function countTokensOpenAIAsync(messages, full = false) {
+ const shouldTokenizeAI21 = oai_settings.chat_completion_source === chat_completion_sources.AI21 && oai_settings.use_ai21_tokenizer;
+ const shouldTokenizeGoogle = oai_settings.chat_completion_source === chat_completion_sources.MAKERSUITE && oai_settings.use_google_tokenizer;
+ let tokenizerEndpoint = '';
+ if (shouldTokenizeAI21) {
+ tokenizerEndpoint = '/api/tokenizers/ai21/count';
+ } else if (shouldTokenizeGoogle) {
+ tokenizerEndpoint = `/api/tokenizers/google/count?model=${getTokenizerModel()}`;
+ } else {
+ tokenizerEndpoint = `/api/tokenizers/openai/count?model=${getTokenizerModel()}`;
+ }
+ const cacheObject = getTokenCacheObject();
+
+ if (!Array.isArray(messages)) {
+ messages = [messages];
+ }
+
+ let token_count = -1;
+
+ for (const message of messages) {
+ const model = getTokenizerModel();
+
+ if (model === 'claude' || shouldTokenizeAI21 || shouldTokenizeGoogle) {
+ full = true;
+ }
+
+ const hash = getStringHash(JSON.stringify(message));
+ const cacheKey = `${model}-${hash}`;
+ const cachedCount = cacheObject[cacheKey];
+
+ if (typeof cachedCount === 'number') {
+ token_count += cachedCount;
+ }
+
+ else {
+ const data = await jQuery.ajax({
+ async: true,
+ type: 'POST', //
+ url: tokenizerEndpoint,
+ data: JSON.stringify([message]),
+ dataType: 'json',
+ contentType: 'application/json',
+ });
+
+ token_count += Number(data.token_count);
+ cacheObject[cacheKey] = Number(data.token_count);
+ }
+ }
+
+ if (!full) token_count -= 2;
+
+ return token_count;
+}
+
/**
* Gets the token cache object for the current chat.
* @returns {Object} Token cache object for the current chat.
@@ -495,13 +649,15 @@ function getTokenCacheObject() {
* Count tokens using the server API.
* @param {string} endpoint API endpoint.
* @param {string} str String to tokenize.
+ * @param {function} [resolve] Promise resolve function.s
* @returns {number} Token count.
*/
-function countTokensFromServer(endpoint, str) {
+function countTokensFromServer(endpoint, str, resolve) {
+ const isAsync = typeof resolve === 'function';
let tokenCount = 0;
jQuery.ajax({
- async: false,
+ async: isAsync,
type: 'POST',
url: endpoint,
data: JSON.stringify({ text: str }),
@@ -513,6 +669,8 @@ function countTokensFromServer(endpoint, str) {
} else {
tokenCount = apiFailureTokenCount(str);
}
+
+ isAsync && resolve(tokenCount);
},
});
@@ -522,13 +680,15 @@ function countTokensFromServer(endpoint, str) {
/**
* Count tokens using the AI provider's API.
* @param {string} str String to tokenize.
+ * @param {function} [resolve] Promise resolve function.
* @returns {number} Token count.
*/
-function countTokensFromKoboldAPI(str) {
+function countTokensFromKoboldAPI(str, resolve) {
+ const isAsync = typeof resolve === 'function';
let tokenCount = 0;
jQuery.ajax({
- async: false,
+ async: isAsync,
type: 'POST',
url: TOKENIZER_URLS[tokenizers.API_KOBOLD].count,
data: JSON.stringify({
@@ -543,6 +703,8 @@ function countTokensFromKoboldAPI(str) {
} else {
tokenCount = apiFailureTokenCount(str);
}
+
+ isAsync && resolve(tokenCount);
},
});
@@ -561,13 +723,15 @@ function getTextgenAPITokenizationParams(str) {
/**
* Count tokens using the AI provider's API.
* @param {string} str String to tokenize.
+ * @param {function} [resolve] Promise resolve function.
* @returns {number} Token count.
*/
-function countTokensFromTextgenAPI(str) {
+function countTokensFromTextgenAPI(str, resolve) {
+ const isAsync = typeof resolve === 'function';
let tokenCount = 0;
jQuery.ajax({
- async: false,
+ async: isAsync,
type: 'POST',
url: TOKENIZER_URLS[tokenizers.API_TEXTGENERATIONWEBUI].count,
data: JSON.stringify(getTextgenAPITokenizationParams(str)),
@@ -579,6 +743,8 @@ function countTokensFromTextgenAPI(str) {
} else {
tokenCount = apiFailureTokenCount(str);
}
+
+ isAsync && resolve(tokenCount);
},
});
@@ -605,12 +771,14 @@ function apiFailureTokenCount(str) {
* Calls the underlying tokenizer model to encode a string to tokens.
* @param {string} endpoint API endpoint.
* @param {string} str String to tokenize.
+ * @param {function} [resolve] Promise resolve function.
* @returns {number[]} Array of token ids.
*/
-function getTextTokensFromServer(endpoint, str) {
+function getTextTokensFromServer(endpoint, str, resolve) {
+ const isAsync = typeof resolve === 'function';
let ids = [];
jQuery.ajax({
- async: false,
+ async: isAsync,
type: 'POST',
url: endpoint,
data: JSON.stringify({ text: str }),
@@ -623,6 +791,8 @@ function getTextTokensFromServer(endpoint, str) {
if (Array.isArray(data.chunks)) {
Object.defineProperty(ids, 'chunks', { value: data.chunks });
}
+
+ isAsync && resolve(ids);
},
});
return ids;
@@ -631,12 +801,14 @@ function getTextTokensFromServer(endpoint, str) {
/**
* Calls the AI provider's tokenize API to encode a string to tokens.
* @param {string} str String to tokenize.
+ * @param {function} [resolve] Promise resolve function.
* @returns {number[]} Array of token ids.
*/
-function getTextTokensFromTextgenAPI(str) {
+function getTextTokensFromTextgenAPI(str, resolve) {
+ const isAsync = typeof resolve === 'function';
let ids = [];
jQuery.ajax({
- async: false,
+ async: isAsync,
type: 'POST',
url: TOKENIZER_URLS[tokenizers.API_TEXTGENERATIONWEBUI].encode,
data: JSON.stringify(getTextgenAPITokenizationParams(str)),
@@ -644,6 +816,7 @@ function getTextTokensFromTextgenAPI(str) {
contentType: 'application/json',
success: function (data) {
ids = data.ids;
+ isAsync && resolve(ids);
},
});
return ids;
@@ -652,13 +825,15 @@ function getTextTokensFromTextgenAPI(str) {
/**
* Calls the AI provider's tokenize API to encode a string to tokens.
* @param {string} str String to tokenize.
+ * @param {function} [resolve] Promise resolve function.
* @returns {number[]} Array of token ids.
*/
-function getTextTokensFromKoboldAPI(str) {
+function getTextTokensFromKoboldAPI(str, resolve) {
+ const isAsync = typeof resolve === 'function';
let ids = [];
jQuery.ajax({
- async: false,
+ async: isAsync,
type: 'POST',
url: TOKENIZER_URLS[tokenizers.API_KOBOLD].encode,
data: JSON.stringify({
@@ -669,6 +844,7 @@ function getTextTokensFromKoboldAPI(str) {
contentType: 'application/json',
success: function (data) {
ids = data.ids;
+ isAsync && resolve(ids);
},
});
@@ -679,13 +855,15 @@ function getTextTokensFromKoboldAPI(str) {
* Calls the underlying tokenizer model to decode token ids to text.
* @param {string} endpoint API endpoint.
* @param {number[]} ids Array of token ids
+ * @param {function} [resolve] Promise resolve function.
* @returns {({ text: string, chunks?: string[] })} Decoded token text as a single string and individual chunks (if available).
*/
-function decodeTextTokensFromServer(endpoint, ids) {
+function decodeTextTokensFromServer(endpoint, ids, resolve) {
+ const isAsync = typeof resolve === 'function';
let text = '';
let chunks = [];
jQuery.ajax({
- async: false,
+ async: isAsync,
type: 'POST',
url: endpoint,
data: JSON.stringify({ ids: ids }),
@@ -694,6 +872,7 @@ function decodeTextTokensFromServer(endpoint, ids) {
success: function (data) {
text = data.text;
chunks = data.chunks;
+ isAsync && resolve({ text, chunks });
},
});
return { text, chunks };
diff --git a/public/scripts/world-info.js b/public/scripts/world-info.js
index a8999e228..09e9d7f34 100644
--- a/public/scripts/world-info.js
+++ b/public/scripts/world-info.js
@@ -5,7 +5,7 @@ import { NOTE_MODULE_NAME, metadata_keys, shouldWIAddPrompt } from './authors-no
import { registerSlashCommand } from './slash-commands.js';
import { isMobile } from './RossAscends-mods.js';
import { FILTER_TYPES, FilterHelper } from './filters.js';
-import { getTokenCount } from './tokenizers.js';
+import { getTokenCountAsync } from './tokenizers.js';
import { power_user } from './power-user.js';
import { getTagKeyForEntity } from './tags.js';
import { resolveVariable } from './variables.js';
@@ -1189,8 +1189,8 @@ function getWorldEntry(name, data, entry) {
// content
const counter = template.find('.world_entry_form_token_counter');
- const countTokensDebounced = debounce(function (counter, value) {
- const numberOfTokens = getTokenCount(value);
+ const countTokensDebounced = debounce(async function (counter, value) {
+ const numberOfTokens = await getTokenCountAsync(value);
$(counter).text(numberOfTokens);
}, 1000);
@@ -2177,7 +2177,7 @@ async function checkWorldInfo(chat, maxContext) {
const newEntries = [...activatedNow]
.sort((a, b) => sortedEntries.indexOf(a) - sortedEntries.indexOf(b));
let newContent = '';
- const textToScanTokens = getTokenCount(allActivatedText);
+ const textToScanTokens = await getTokenCountAsync(allActivatedText);
const probabilityChecksBefore = failedProbabilityChecks.size;
filterByInclusionGroups(newEntries, allActivatedEntries);
@@ -2194,7 +2194,7 @@ async function checkWorldInfo(chat, maxContext) {
newContent += `${substituteParams(entry.content)}\n`;
- if (textToScanTokens + getTokenCount(newContent) >= budget) {
+ if ((textToScanTokens + (await getTokenCountAsync(newContent))) >= budget) {
console.debug('WI budget reached, stopping');
if (world_info_overflow_alert) {
console.log('Alerting');
diff --git a/server.js b/server.js
index 81c1a3119..172b779f1 100644
--- a/server.js
+++ b/server.js
@@ -498,12 +498,14 @@ const setupTasks = async function () {
await statsEndpoint.init();
const cleanupPlugins = await loadPlugins();
+ const consoleTitle = process.title;
const exitProcess = async () => {
statsEndpoint.onExit();
if (typeof cleanupPlugins === 'function') {
await cleanupPlugins();
}
+ setWindowTitle(consoleTitle);
process.exit();
};
@@ -520,6 +522,8 @@ const setupTasks = async function () {
if (autorun) open(autorunUrl.toString());
+ setWindowTitle('SillyTavern WebServer');
+
console.log(color.green('SillyTavern is listening on: ' + tavernUrl));
if (listen) {
@@ -561,6 +565,19 @@ if (listen && !getConfigValue('whitelistMode', true) && !basicAuthMode) {
}
}
+/**
+ * Set the title of the terminal window
+ * @param {string} title Desired title for the window
+ */
+function setWindowTitle(title) {
+ if (process.platform === 'win32') {
+ process.title = title;
+ }
+ else {
+ process.stdout.write(`\x1b]2;${title}\x1b\x5c`);
+ }
+}
+
if (cliArguments.ssl) {
https.createServer(
{
diff --git a/src/constants.js b/src/constants.js
index 918374eab..e7831bf48 100644
--- a/src/constants.js
+++ b/src/constants.js
@@ -243,8 +243,8 @@ const OLLAMA_KEYS = [
'mirostat_eta',
];
-const AVATAR_WIDTH = 400;
-const AVATAR_HEIGHT = 600;
+const AVATAR_WIDTH = 512;
+const AVATAR_HEIGHT = 768;
const OPENROUTER_HEADERS = {
'HTTP-Referer': 'https://sillytavern.app',
diff --git a/src/endpoints/backends/text-completions.js b/src/endpoints/backends/text-completions.js
index 8e0ebba8d..22806cbf0 100644
--- a/src/endpoints/backends/text-completions.js
+++ b/src/endpoints/backends/text-completions.js
@@ -516,7 +516,7 @@ llamacpp.post('/slots', jsonParser, async function (request, response) {
const baseUrl = trimV1(request.body.server_url);
let fetchResponse;
- if (request.body.action === "info") {
+ if (request.body.action === 'info') {
fetchResponse = await fetch(`${baseUrl}/slots`, {
method: 'GET',
timeout: 0,
@@ -525,16 +525,16 @@ llamacpp.post('/slots', jsonParser, async function (request, response) {
if (!/^\d+$/.test(request.body.id_slot)) {
return response.sendStatus(400);
}
- if (request.body.action !== "erase" && !request.body.filename) {
+ if (request.body.action !== 'erase' && !request.body.filename) {
return response.sendStatus(400);
}
-
+
fetchResponse = await fetch(`${baseUrl}/slots/${request.body.id_slot}?action=${request.body.action}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
timeout: 0,
body: JSON.stringify({
- filename: request.body.action !== "erase" ? `${request.body.filename}` : undefined,
+ filename: request.body.action !== 'erase' ? `${request.body.filename}` : undefined,
}),
});
}
diff --git a/src/endpoints/stable-diffusion.js b/src/endpoints/stable-diffusion.js
index 3902fa3ce..52d2d2957 100644
--- a/src/endpoints/stable-diffusion.js
+++ b/src/endpoints/stable-diffusion.js
@@ -685,14 +685,16 @@ drawthings.post('/generate', jsonParser, async (request, response) => {
url.pathname = '/sdapi/v1/txt2img';
const body = { ...request.body };
+ const auth = getBasicAuthHeader(request.body.auth);
delete body.url;
+ delete body.auth;
const result = await fetch(url, {
method: 'POST',
body: JSON.stringify(body),
headers: {
'Content-Type': 'application/json',
- 'Authorization': getBasicAuthHeader(request.body.auth),
+ 'Authorization': auth,
},
timeout: 0,
});
diff --git a/src/middleware/whitelist.js b/src/middleware/whitelist.js
index 87d5ac5a5..2ebc5e48f 100644
--- a/src/middleware/whitelist.js
+++ b/src/middleware/whitelist.js
@@ -19,6 +19,22 @@ if (fs.existsSync(whitelistPath)) {
}
}
+function getForwardedIp(req) {
+ // Check if X-Real-IP is available
+ if (req.headers['x-real-ip']) {
+ return req.headers['x-real-ip'];
+ }
+
+ // Check for X-Forwarded-For and parse if available
+ if (req.headers['x-forwarded-for']) {
+ const ipList = req.headers['x-forwarded-for'].split(',').map(ip => ip.trim());
+ return ipList[0];
+ }
+
+ // If none of the headers are available, return undefined
+ return undefined;
+}
+
function getIpFromRequest(req) {
let clientIp = req.connection.remoteAddress;
let ip = ipaddr.parse(clientIp);
@@ -41,6 +57,7 @@ function getIpFromRequest(req) {
function whitelistMiddleware(listen) {
return function (req, res, next) {
const clientIp = getIpFromRequest(req);
+ const forwardedIp = getForwardedIp(req);
if (listen && !knownIPs.has(clientIp)) {
const userAgent = req.headers['user-agent'];
@@ -58,9 +75,13 @@ function whitelistMiddleware(listen) {
}
//clientIp = req.connection.remoteAddress.split(':').pop();
- if (whitelistMode === true && !whitelist.some(x => ipMatching.matches(clientIp, ipMatching.getMatch(x)))) {
- console.log(color.red('Forbidden: Connection attempt from ' + clientIp + '. If you are attempting to connect, please add your IP address in whitelist or disable whitelist mode in config.yaml in root of SillyTavern folder.\n'));
- return res.status(403).send('Forbidden: Connection attempt from ' + clientIp + '. If you are attempting to connect, please add your IP address in whitelist or disable whitelist mode in config.yaml in root of SillyTavern folder.');
+ if (whitelistMode === true && !whitelist.some(x => ipMatching.matches(clientIp, ipMatching.getMatch(x)))
+ || forwardedIp && whitelistMode === true && !whitelist.some(x => ipMatching.matches(forwardedIp, ipMatching.getMatch(x)))
+ ) {
+ // Log the connection attempt with real IP address
+ const ipDetails = forwardedIp ? `${clientIp} (forwarded from ${forwardedIp})` : clientIp;
+ console.log(color.red('Forbidden: Connection attempt from ' + ipDetails + '. If you are attempting to connect, please add your IP address in whitelist or disable whitelist mode in config.yaml in root of SillyTavern folder.\n'));
+ return res.status(403).send('Forbidden: Connection attempt from ' + ipDetails + '. If you are attempting to connect, please add your IP address in whitelist or disable whitelist mode in config.yaml in root of SillyTavern folder.');
}
next();
};