diff --git a/public/scripts/slash-commands.js b/public/scripts/slash-commands.js
index d9d27f0db..6dcac6a80 100644
--- a/public/scripts/slash-commands.js
+++ b/public/scripts/slash-commands.js
@@ -52,7 +52,7 @@ import { chat_completion_sources, oai_settings, setupChatCompletionPromptManager
import { autoSelectPersona, retriggerFirstMessageOnEmptyChat, setPersonaLockState, togglePersonaLock, user_avatar } from './personas.js';
import { addEphemeralStoppingString, chat_styles, flushEphemeralStoppingStrings, power_user } from './power-user.js';
import { SERVER_INPUTS, textgen_types, textgenerationwebui_settings } from './textgen-settings.js';
-import { decodeTextTokens, getFriendlyTokenizerName, getTextTokens, getTokenCountAsync } from './tokenizers.js';
+import { decodeTextTokens, getAvailableTokenizers, getFriendlyTokenizerName, getTextTokens, getTokenCountAsync, selectTokenizer } from './tokenizers.js';
import { debounce, delay, isFalseBoolean, isTrueBoolean, showFontAwesomePicker, stringToRange, trimToEndSentence, trimToStartSentence, waitUntilCondition } from './utils.js';
import { registerVariableCommands, resolveVariable } from './variables.js';
import { background_settings } from './backgrounds.js';
@@ -1558,6 +1558,28 @@ export function initDefaultSlashCommands() {
`,
}));
+ SlashCommandParser.addCommandObject(SlashCommand.fromProps({
+ name: 'tokenizer',
+ callback: selectTokenizerCallback,
+ returns: 'current tokenizer',
+ unnamedArgumentList: [
+ SlashCommandArgument.fromProps({
+ description: 'tokenizer name',
+ typeList: [ARGUMENT_TYPE.STRING],
+ enumList: getAvailableTokenizers().map(tokenizer =>
+ new SlashCommandEnumValue(tokenizer.tokenizerKey, tokenizer.tokenizerName, enumTypes.enum, enumIcons.default)),
+ }),
+ ],
+ helpString: `
+
+ Selects tokenizer by name. Gets the current tokenizer if no name is provided.
+
+
+
Available tokenizers:
+
${getAvailableTokenizers().map(t => t.tokenizerKey).join(', ')}
+
+ `,
+ }));
registerVariableCommands();
}
@@ -3562,6 +3584,27 @@ async function setApiUrlCallback({ api = null, connect = 'true' }, url) {
return textgenerationwebui_settings.server_urls[type] ?? '';
}
+async function selectTokenizerCallback(_, name) {
+ if (!name) {
+ return getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === power_user.tokenizer)?.tokenizerKey ?? '';
+ }
+
+ const tokenizers = getAvailableTokenizers();
+ const fuse = new Fuse(tokenizers, { keys: ['tokenizerKey', 'tokenizerName'] });
+ const result = fuse.search(name);
+
+ if (result.length === 0) {
+ toastr.warning(`Tokenizer "${name}" not found`);
+ return '';
+ }
+
+ /** @type {import('./tokenizers.js').Tokenizer} */
+ const foundTokenizer = result[0].item;
+ selectTokenizer(foundTokenizer.tokenizerId);
+
+ return foundTokenizer.tokenizerKey;
+}
+
export let isExecutingCommandsFromChatInput = false;
export let commandsFromChatInputAbortController;
diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js
index c27775acb..e72a44ad6 100644
--- a/public/scripts/tokenizers.js
+++ b/public/scripts/tokenizers.js
@@ -147,10 +147,46 @@ async function resetTokenCache() {
}
}
+/**
+ * @typedef {object} Tokenizer
+ * @property {number} tokenizerId - The id of the tokenizer option
+ * @property {string} tokenizerKey - Internal name/key of the tokenizer
+ * @property {string} tokenizerName - Human-readable detailed name of the tokenizer (as displayed in the UI)
+ */
+
+/**
+ * Gets all tokenizers available to the user.
+ * @returns {Tokenizer[]} Tokenizer info.
+ */
+export function getAvailableTokenizers() {
+ const tokenizerOptions = $('#tokenizer').find('option').toArray();
+ return tokenizerOptions.map(tokenizerOption => ({
+ tokenizerId: Number(tokenizerOption.value),
+ tokenizerKey: Object.entries(tokenizers).find(([_, value]) => value === Number(tokenizerOption.value))[0].toLocaleLowerCase(),
+ tokenizerName: tokenizerOption.text,
+ }))
+}
+
+/**
+ * Selects tokenizer if not already selected.
+ * @param {number} tokenizerId Tokenizer ID.
+ */
+export function selectTokenizer(tokenizerId) {
+ if (tokenizerId !== power_user.tokenizer) {
+ const tokenizer = getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === tokenizerId);
+ if (!tokenizer) {
+ console.warn('Failed to find tokenizer with id', tokenizerId);
+ return;
+ }
+ $('#tokenizer').val(tokenizer.tokenizerId).trigger('change');
+ toastr.info(`Tokenizer: "${tokenizer.tokenizerName}" selected`);
+ }
+}
+
/**
* Gets the friendly name of the current tokenizer.
* @param {string} forApi API to get the tokenizer for. Defaults to the main API.
- * @returns { { tokenizerName: string, tokenizerId: number } } Tokenizer info
+ * @returns {Tokenizer} Tokenizer info
*/
export function getFriendlyTokenizerName(forApi) {
if (!forApi) {
@@ -185,7 +221,9 @@ export function getFriendlyTokenizerName(forApi) {
? tokenizers.OPENAI
: tokenizerId;
- return { tokenizerName, tokenizerId };
+ const tokenizerKey = Object.entries(tokenizers).find(([_, value]) => value === tokenizerId)[0].toLocaleLowerCase();
+
+ return { tokenizerName, tokenizerKey, tokenizerId };
}
/**