diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js index d9d27f0db..6dcac6a80 100644 --- a/public/scripts/slash-commands.js +++ b/public/scripts/slash-commands.js @@ -52,7 +52,7 @@ import { chat_completion_sources, oai_settings, setupChatCompletionPromptManager import { autoSelectPersona, retriggerFirstMessageOnEmptyChat, setPersonaLockState, togglePersonaLock, user_avatar } from './personas.js'; import { addEphemeralStoppingString, chat_styles, flushEphemeralStoppingStrings, power_user } from './power-user.js'; import { SERVER_INPUTS, textgen_types, textgenerationwebui_settings } from './textgen-settings.js'; -import { decodeTextTokens, getFriendlyTokenizerName, getTextTokens, getTokenCountAsync } from './tokenizers.js'; +import { decodeTextTokens, getAvailableTokenizers, getFriendlyTokenizerName, getTextTokens, getTokenCountAsync, selectTokenizer } from './tokenizers.js'; import { debounce, delay, isFalseBoolean, isTrueBoolean, showFontAwesomePicker, stringToRange, trimToEndSentence, trimToStartSentence, waitUntilCondition } from './utils.js'; import { registerVariableCommands, resolveVariable } from './variables.js'; import { background_settings } from './backgrounds.js'; @@ -1558,6 +1558,28 @@ export function initDefaultSlashCommands() { `, })); + SlashCommandParser.addCommandObject(SlashCommand.fromProps({ + name: 'tokenizer', + callback: selectTokenizerCallback, + returns: 'current tokenizer', + unnamedArgumentList: [ + SlashCommandArgument.fromProps({ + description: 'tokenizer name', + typeList: [ARGUMENT_TYPE.STRING], + enumList: getAvailableTokenizers().map(tokenizer => + new SlashCommandEnumValue(tokenizer.tokenizerKey, tokenizer.tokenizerName, enumTypes.enum, enumIcons.default)), + }), + ], + helpString: ` +
+ Selects tokenizer by name. Gets the current tokenizer if no name is provided. +
+
+ Available tokenizers: +
${getAvailableTokenizers().map(t => t.tokenizerKey).join(', ')}
+
+ `, + })); registerVariableCommands(); } @@ -3562,6 +3584,27 @@ async function setApiUrlCallback({ api = null, connect = 'true' }, url) { return textgenerationwebui_settings.server_urls[type] ?? ''; } +async function selectTokenizerCallback(_, name) { + if (!name) { + return getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === power_user.tokenizer)?.tokenizerKey ?? ''; + } + + const tokenizers = getAvailableTokenizers(); + const fuse = new Fuse(tokenizers, { keys: ['tokenizerKey', 'tokenizerName'] }); + const result = fuse.search(name); + + if (result.length === 0) { + toastr.warning(`Tokenizer "${name}" not found`); + return ''; + } + + /** @type {import('./tokenizers.js').Tokenizer} */ + const foundTokenizer = result[0].item; + selectTokenizer(foundTokenizer.tokenizerId); + + return foundTokenizer.tokenizerKey; +} + export let isExecutingCommandsFromChatInput = false; export let commandsFromChatInputAbortController; diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js index c27775acb..e72a44ad6 100644 --- a/public/scripts/tokenizers.js +++ b/public/scripts/tokenizers.js @@ -147,10 +147,46 @@ async function resetTokenCache() { } } +/** + * @typedef {object} Tokenizer + * @property {number} tokenizerId - The id of the tokenizer option + * @property {string} tokenizerKey - Internal name/key of the tokenizer + * @property {string} tokenizerName - Human-readable detailed name of the tokenizer (as displayed in the UI) + */ + +/** + * Gets all tokenizers available to the user. + * @returns {Tokenizer[]} Tokenizer info. + */ +export function getAvailableTokenizers() { + const tokenizerOptions = $('#tokenizer').find('option').toArray(); + return tokenizerOptions.map(tokenizerOption => ({ + tokenizerId: Number(tokenizerOption.value), + tokenizerKey: Object.entries(tokenizers).find(([_, value]) => value === Number(tokenizerOption.value))[0].toLocaleLowerCase(), + tokenizerName: tokenizerOption.text, + })) +} + +/** + * Selects tokenizer if not already selected. + * @param {number} tokenizerId Tokenizer ID. + */ +export function selectTokenizer(tokenizerId) { + if (tokenizerId !== power_user.tokenizer) { + const tokenizer = getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === tokenizerId); + if (!tokenizer) { + console.warn('Failed to find tokenizer with id', tokenizerId); + return; + } + $('#tokenizer').val(tokenizer.tokenizerId).trigger('change'); + toastr.info(`Tokenizer: "${tokenizer.tokenizerName}" selected`); + } +} + /** * Gets the friendly name of the current tokenizer. * @param {string} forApi API to get the tokenizer for. Defaults to the main API. - * @returns { { tokenizerName: string, tokenizerId: number } } Tokenizer info + * @returns {Tokenizer} Tokenizer info */ export function getFriendlyTokenizerName(forApi) { if (!forApi) { @@ -185,7 +221,9 @@ export function getFriendlyTokenizerName(forApi) { ? tokenizers.OPENAI : tokenizerId; - return { tokenizerName, tokenizerId }; + const tokenizerKey = Object.entries(tokenizers).find(([_, value]) => value === tokenizerId)[0].toLocaleLowerCase(); + + return { tokenizerName, tokenizerKey, tokenizerId }; } /**