diff --git a/public/script.js b/public/script.js index 4599e1cdc..357976ee5 100644 --- a/public/script.js +++ b/public/script.js @@ -212,7 +212,7 @@ import { selectContextPreset, } from './scripts/instruct-mode.js'; import { initLocales, t, translate } from './scripts/i18n.js'; -import { getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache, selectTokenizer, TOKENIZER_NAME_MAP, tokenizers } from './scripts/tokenizers.js'; +import { getAvailableTokenizers, getFriendlyTokenizerName, getTokenCount, getTokenCountAsync, getTokenizerModel, initTokenizers, saveTokenCache, selectTokenizer } from './scripts/tokenizers.js'; import { user_avatar, getUserAvatars, @@ -8453,11 +8453,11 @@ async function selectInstructCallback(_, name) { async function selectTokenizerCallback(_, name) { if (!name) { - return TOKENIZER_NAME_MAP[power_user.tokenizer]; + return getFriendlyTokenizerName(main_api).tokenizerName; } - const tokenizerNames = Object.values(TOKENIZER_NAME_MAP); - const fuse = new Fuse(tokenizerNames); + const tokenizers = getAvailableTokenizers(); + const fuse = new Fuse(tokenizers, { keys: ['tokenizerName'] }); const result = fuse.search(name); if (result.length === 0) { @@ -8465,9 +8465,9 @@ async function selectTokenizerCallback(_, name) { return ''; } - const foundName = result[0].item; - selectTokenizer(tokenizers[foundName.toUpperCase()]); - return foundName; + const foundTokenizer = result[0].item; + selectTokenizer(foundTokenizer.tokenizerName, foundTokenizer.tokenizerId); + return foundTokenizer; } async function enableInstructCallback() { @@ -9122,8 +9122,8 @@ jQuery(async function () { SlashCommandArgument.fromProps({ description: 'tokenizer name', typeList: [ARGUMENT_TYPE.STRING], - enumList: Object.values(TOKENIZER_NAME_MAP).map(tokenizer => - new SlashCommandEnumValue(tokenizer, null, enumTypes.enum, enumIcons.default)), + enumList: getAvailableTokenizers().map(tokenizer => + new SlashCommandEnumValue(tokenizer.tokenizerName, null, enumTypes.enum, enumIcons.default)), }), ], helpString: ` @@ -9132,7 +9132,7 @@ jQuery(async function () {
Available tokenizers: -
${Object.values(TOKENIZER_NAME_MAP).join(', ')}
+
${getAvailableTokenizers().map(t => t.tokenizerName).join(', ')}
` })); diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js index 5732706d3..c4411bd3f 100644 --- a/public/scripts/tokenizers.js +++ b/public/scripts/tokenizers.js @@ -148,20 +148,26 @@ async function resetTokenCache() { } /** - * Maps tokenizer IDs to their names. - * @example { 0: 'none', 1: 'gpt2', ... } + * Gets all tokenizers available to the user. + * @returns { { tokenizerName: string, tokenizerId: number }[] } Tokenizer info. */ -export const TOKENIZER_NAME_MAP = Object.fromEntries( - Object.entries(tokenizers).map(([name, id]) => [id, name.toLowerCase()])); +export function getAvailableTokenizers() { + const tokenizerOptions = $('#tokenizer').find('option').toArray(); + return tokenizerOptions.map(tokenizerOption => ({ + tokenizerName: tokenizerOption.text, + tokenizerId: Number(tokenizerOption.value), + })) +} /** * Selects tokenizer if not already selected. + * @param {string} tokenizerName Tokenizer name. * @param {number} tokenizerId Tokenizer ID. */ -export function selectTokenizer(tokenizerId) { +export function selectTokenizer(tokenizerName, tokenizerId) { if (tokenizerId !== power_user.tokenizer) { $('#tokenizer').val(tokenizerId).trigger('change'); - toastr.info(`Tokenizer: "${TOKENIZER_NAME_MAP[tokenizerId]}" selected`); + toastr.info(`Tokenizer: "${tokenizerName}" selected`); } }