diff --git a/public/script.js b/public/script.js index 5b89ac5f4..273852398 100644 --- a/public/script.js +++ b/public/script.js @@ -8453,11 +8453,11 @@ async function selectInstructCallback(_, name) { async function selectTokenizerCallback(_, name) { if (!name) { - return getFriendlyTokenizerName(main_api).tokenizerName; + return getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === power_user.tokenizer)?.tokenizerKey ?? ''; } const tokenizers = getAvailableTokenizers(); - const fuse = new Fuse(tokenizers, { keys: ['tokenizerName'] }); + const fuse = new Fuse(tokenizers, { keys: ['tokenizerKey', 'tokenizerName'] }); const result = fuse.search(name); if (result.length === 0) { @@ -8465,9 +8465,11 @@ async function selectTokenizerCallback(_, name) { return ''; } + /** @type {import('./scripts/tokenizers.js').Tokenizer} */ const foundTokenizer = result[0].item; - selectTokenizer(foundTokenizer.tokenizerName, foundTokenizer.tokenizerId); - return foundTokenizer.tokenizerName; + selectTokenizer(foundTokenizer.tokenizerId); + + return foundTokenizer.tokenizerKey; } async function enableInstructCallback() { @@ -9123,7 +9125,7 @@ jQuery(async function () { description: 'tokenizer name', typeList: [ARGUMENT_TYPE.STRING], enumList: getAvailableTokenizers().map(tokenizer => - new SlashCommandEnumValue(tokenizer.tokenizerName, null, enumTypes.enum, enumIcons.default)), + new SlashCommandEnumValue(tokenizer.tokenizerKey, tokenizer.tokenizerName, enumTypes.enum, enumIcons.default)), }), ], helpString: ` @@ -9132,7 +9134,7 @@ jQuery(async function () {
Available tokenizers: -
${getAvailableTokenizers().map(t => t.tokenizerName).join(', ')}
+
${getAvailableTokenizers().map(t => t.tokenizerKey).join(', ')}
` })); diff --git a/public/scripts/tokenizers.js b/public/scripts/tokenizers.js index c4411bd3f..e72a44ad6 100644 --- a/public/scripts/tokenizers.js +++ b/public/scripts/tokenizers.js @@ -147,34 +147,46 @@ async function resetTokenCache() { } } +/** + * @typedef {object} Tokenizer + * @property {number} tokenizerId - The id of the tokenizer option + * @property {string} tokenizerKey - Internal name/key of the tokenizer + * @property {string} tokenizerName - Human-readable detailed name of the tokenizer (as displayed in the UI) + */ + /** * Gets all tokenizers available to the user. - * @returns { { tokenizerName: string, tokenizerId: number }[] } Tokenizer info. + * @returns {Tokenizer[]} Tokenizer info. */ export function getAvailableTokenizers() { const tokenizerOptions = $('#tokenizer').find('option').toArray(); return tokenizerOptions.map(tokenizerOption => ({ - tokenizerName: tokenizerOption.text, tokenizerId: Number(tokenizerOption.value), + tokenizerKey: Object.entries(tokenizers).find(([_, value]) => value === Number(tokenizerOption.value))[0].toLocaleLowerCase(), + tokenizerName: tokenizerOption.text, })) } /** * Selects tokenizer if not already selected. - * @param {string} tokenizerName Tokenizer name. * @param {number} tokenizerId Tokenizer ID. */ -export function selectTokenizer(tokenizerName, tokenizerId) { +export function selectTokenizer(tokenizerId) { if (tokenizerId !== power_user.tokenizer) { - $('#tokenizer').val(tokenizerId).trigger('change'); - toastr.info(`Tokenizer: "${tokenizerName}" selected`); + const tokenizer = getAvailableTokenizers().find(tokenizer => tokenizer.tokenizerId === tokenizerId); + if (!tokenizer) { + console.warn('Failed to find tokenizer with id', tokenizerId); + return; + } + $('#tokenizer').val(tokenizer.tokenizerId).trigger('change'); + toastr.info(`Tokenizer: "${tokenizer.tokenizerName}" selected`); } } /** * Gets the friendly name of the current tokenizer. * @param {string} forApi API to get the tokenizer for. Defaults to the main API. - * @returns { { tokenizerName: string, tokenizerId: number } } Tokenizer info + * @returns {Tokenizer} Tokenizer info */ export function getFriendlyTokenizerName(forApi) { if (!forApi) { @@ -209,7 +221,9 @@ export function getFriendlyTokenizerName(forApi) { ? tokenizers.OPENAI : tokenizerId; - return { tokenizerName, tokenizerId }; + const tokenizerKey = Object.entries(tokenizers).find(([_, value]) => value === tokenizerId)[0].toLocaleLowerCase(); + + return { tokenizerName, tokenizerKey, tokenizerId }; } /**